Compare commits

..
Author SHA1 Message Date
Anthony Stirling f3b170f96a i18n 2026-05-15 12:44:33 +01:00
Anthony Stirling d93c2798e8 fixes 2026-05-15 12:42:57 +01:00
Anthony Stirling a3dd9a39ff improve merge 2026-05-15 12:20:34 +01:00
366 changed files with 7089 additions and 19119 deletions
+1 -1
View File
@@ -22,7 +22,7 @@ runs:
steps:
- name: Generate a GitHub App Token
id: generate-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
with:
app-id: ${{ inputs.app-id }}
private-key: ${{ inputs.private-key }}
+1 -1
View File
@@ -1,6 +1,6 @@
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
pkgname=stirling-pdf-desktop
pkgver=2.11.0
pkgver=2.10.1
pkgrel=1
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
arch=('x86_64')
+1 -1
View File
@@ -1,6 +1,6 @@
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
pkgname=stirling-pdf-server-bin
pkgver=2.11.0
pkgver=2.10.1
pkgrel=1
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
arch=('any')
+3 -3
View File
@@ -35,7 +35,7 @@ jobs:
pr_ref: ${{ steps.resolve.outputs.ref }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -111,7 +111,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -359,7 +359,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -70,7 +70,7 @@ jobs:
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -183,7 +183,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -474,7 +474,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+3 -3
View File
@@ -26,7 +26,7 @@ jobs:
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -70,7 +70,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -106,7 +106,7 @@ jobs:
- name: Publish stirling-pdf-desktop to AUR
if: ${{ github.event_name == 'release' || inputs.dry_run == false }}
uses: KSXGitHub/github-actions-deploy-aur@v4.1.3
uses: KSXGitHub/github-actions-deploy-aur@v4.1.2
with:
pkgname: stirling-pdf-desktop
pkgbuild: .github/aur/stirling-pdf-desktop/PKGBUILD
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
spring-security: [true, false]
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout repository
@@ -35,7 +35,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/wrapper
+5 -13
View File
@@ -44,7 +44,7 @@ jobs:
SYSTEM_ENABLEANALYTICS: "false"
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout repository
@@ -63,15 +63,7 @@ jobs:
- name: Install Task
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
- name: Install Playwright (chromium only)
run: task e2e:install -- chromium
- name: Build frontend (needed for playwright's vite preview webServer)
# Enterprise tests target :8080 (Spring Boot's bundled frontend), but
# playwright's webServer config still launches `vite preview --port 5173`
# before any test run, which needs dist/ to exist. VITE_BUILD_FOR_PREVIEW
# forces absolute asset paths so vite preview can serve deep SPA routes.
env:
VITE_BUILD_FOR_PREVIEW: "1"
run: task frontend:build
run: task frontend:test:e2e:install -- chromium
- name: Resolve kubernetes.docker.internal to localhost
# The compose stacks set KC_HOSTNAME=kubernetes.docker.internal so
@@ -148,7 +140,7 @@ jobs:
wait_for_backend
- name: Run enterprise OAuth Playwright tests
id: oauth-tests
run: task e2e:enterprise -- --grep "OAuth"
run: task frontend:test:e2e -- --project=enterprise --grep "OAuth"
- name: Stop backend + tear down OAuth Keycloak
if: always()
run: |
@@ -221,7 +213,7 @@ jobs:
wait_for_backend
- name: Run enterprise SAML Playwright tests
id: saml-tests
run: task e2e:enterprise -- --grep "SAML"
run: task frontend:test:e2e -- --project=enterprise --grep "SAML"
- name: Stop backend + tear down SAML Keycloak
if: always()
run: |
@@ -251,7 +243,7 @@ jobs:
wait_for_backend
- name: Run enterprise feature Playwright tests
id: feature-tests
run: task e2e:enterprise -- --grep "Enterprise license"
run: task frontend:test:e2e -- --project=enterprise --grep "Enterprise license"
- name: Print backend log on failure
if: failure()
run: |
+1 -1
View File
@@ -46,7 +46,7 @@ jobs:
proprietary: ${{ steps.changes.outputs.proprietary }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout repository
+2 -2
View File
@@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -27,7 +27,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/wrapper
+2 -2
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -28,7 +28,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/wrapper
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
pull-requests: write # Allow writing to pull requests
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+2 -2
View File
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -39,7 +39,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/wrapper
+40 -8
View File
@@ -15,7 +15,7 @@ jobs:
timeout-minutes: 30
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout repository
@@ -34,26 +34,58 @@ jobs:
- name: Install Task
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
- name: Install Playwright (chromium only)
run: task e2e:install -- chromium
- name: Build frontend (production bundle for vite preview)
run: task frontend:test:e2e:install -- chromium
- name: Start Spring Boot backend (background)
env:
VITE_BUILD_FOR_PREVIEW: "1"
run: task frontend:build
# Suppress the analytics opt-in modal that fires on first admin login when
# enableAnalytics is null (see Onboarding.tsx). The modal renders a Mantine
# overlay that intercepts pointer events on every tool page until dismissed,
# which causes every "click run button" assertion in the live suite to fail.
SYSTEM_ENABLEANALYTICS: "false"
# NOTE: SECURITY_INITIALLOGIN_USERNAME/PASSWORD are intentionally NOT set.
# The live-setup project's bootstrap spec performs the real first-login
# flow against the backend's default admin/stirling user, exercising the
# forced-password-change UI and leaving the DB at admin/adminadmin for
# the rest of the live suite. This is both real coverage of the first-
# login flow and a stronger seed than env-var-driven user creation.
run: |
nohup ./gradlew :stirling-pdf:bootRun > /tmp/backend.log 2>&1 &
echo $! > /tmp/backend.pid
- name: Wait for backend to become ready
run: |
start=$SECONDS
# 300 iterations × 2s = 10 minute ceiling
for i in $(seq 1 300); do
if curl -fsS http://localhost:8080/api/v1/info/status >/dev/null 2>&1; then
echo "Backend up after $((SECONDS - start))s"
exit 0
fi
sleep 2
done
echo "Backend did not become ready in $((SECONDS - start))s"
tail -200 /tmp/backend.log || true
exit 1
- name: Run live E2E tests (chromium)
id: live-tests
run: task e2e:live
run: task frontend:test:e2e -- --project=live
- name: Print backend log on failure
if: failure() && steps.live-tests.conclusion == 'failure'
run: |
echo "::group::Spring Boot backend log (last 500 lines)"
tail -500 .test-state/playwright/backend.log || echo "no backend log found"
tail -500 /tmp/backend.log || echo "no backend log found"
echo "::endgroup::"
- name: Stop backend
if: always()
run: |
if [ -f /tmp/backend.pid ]; then
kill "$(cat /tmp/backend.pid)" 2>/dev/null || true
fi
- name: Upload backend log
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: backend-log-live-${{ github.run_id }}
path: .test-state/playwright/backend.log
path: /tmp/backend.log
retention-days: 7
- name: Upload Playwright report
if: always()
+3 -7
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout repository
@@ -28,13 +28,9 @@ jobs:
- name: Install Task
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
- name: Install Playwright (chromium only)
run: task e2e:install -- chromium
- name: Build frontend (production bundle for vite preview)
env:
VITE_BUILD_FOR_PREVIEW: "1"
run: task frontend:build
run: task frontend:test:e2e:install -- chromium
- name: Run stubbed E2E tests (chromium)
run: task e2e:stubbed -- --workers=3
run: task frontend:test:e2e -- --project=stubbed --workers=3
- name: Upload Playwright report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -28,7 +28,7 @@ jobs:
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -52,7 +52,7 @@ jobs:
repository-projects: write # Required for enabling automerge
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -316,7 +316,7 @@ jobs:
repository-projects: write # Required for enabling automerge
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -21
View File
@@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout repository
@@ -85,26 +85,6 @@ jobs:
echo "to see what still needs fixing manually."
echo "============================================"
exit 1
- name: Remove frontend check comment on success
if: steps.frontend-check.outcome == 'success' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const marker = '<!-- frontend-check -->';
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
});
}
- name: Upload frontend build artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
+1 -1
View File
@@ -15,7 +15,7 @@ jobs:
issues: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+5 -5
View File
@@ -44,7 +44,7 @@ jobs:
version: ${{ steps.versionNumber.outputs.versionNumber }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -57,7 +57,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependencies
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/caches
@@ -127,7 +127,7 @@ jobs:
file_suffix: "-server"
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -191,7 +191,7 @@ jobs:
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
allowed-endpoints: >
@@ -606,7 +606,7 @@ jobs:
contents: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+3 -3
View File
@@ -18,7 +18,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -35,10 +35,10 @@ jobs:
- name: Install Task
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
- name: Install all Playwright browsers
run: task e2e:install
run: task frontend:test:e2e:install
- name: Run E2E tests (all browsers)
run: task e2e:cross-browser
run: task frontend:test:e2e
- name: Upload Playwright report
if: always()
+2 -2
View File
@@ -28,7 +28,7 @@ jobs:
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -79,7 +79,7 @@ jobs:
contents: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
PIP_DISABLE_PIP_VERSION_CHECK: "1"
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -43,7 +43,7 @@ jobs:
echo "version=${VERSION}" >> $GITHUB_OUTPUT
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+9 -165
View File
@@ -2,22 +2,6 @@ name: Push Docker Image with VersionNumber
on:
workflow_dispatch:
inputs:
build_main_app:
description: "Build & push the main Stirling-PDF image (latest, fat, ultra-lite)."
required: false
type: boolean
default: true
build_unoserver:
description: "Build & push the standalone stirling-unoserver image."
required: false
type: boolean
default: true
force_unoserver_rebuild:
description: "Rebuild stirling-unoserver even if its source hash is unchanged."
required: false
type: boolean
default: false
push:
branches:
- master
@@ -47,13 +31,9 @@ jobs:
permissions:
packages: write
id-token: write
# On push events these stay 'true'; on workflow_dispatch they follow the inputs.
env:
RUN_MAIN_APP: ${{ github.event_name != 'workflow_dispatch' || inputs.build_main_app }}
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -66,7 +46,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependencies
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/caches
@@ -128,7 +108,6 @@ jobs:
- name: Generate tags for latest
id: meta
if: env.RUN_MAIN_APP == 'true'
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: |
@@ -143,8 +122,6 @@ jobs:
- name: Build and push Unified Dockerfile (latest variant)
id: build-push-latest
# Empty-tag guard: build-push-action errors when asked to push with no tags.
if: env.RUN_MAIN_APP == 'true' && steps.meta.outputs.tags != ''
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
@@ -163,7 +140,7 @@ jobs:
sbom: true
- name: Sign regular images
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-latest.outputs.digest != ''
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
env:
DIGEST: ${{ steps.build-push-latest.outputs.digest }}
TAGS: ${{ steps.meta.outputs.tags }}
@@ -179,7 +156,7 @@ jobs:
- name: Generate tags for latest-fat
id: meta-fat
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
with:
images: |
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
@@ -193,7 +170,7 @@ jobs:
- name: Build and push Unified Dockerfile (fat variant)
id: build-push-fat
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-fat.outputs.tags != ''
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
@@ -209,7 +186,7 @@ jobs:
sbom: true
- name: Sign fat images
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-fat.outputs.digest != ''
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
env:
DIGEST: ${{ steps.build-push-fat.outputs.digest }}
TAGS: ${{ steps.meta-fat.outputs.tags }}
@@ -223,7 +200,7 @@ jobs:
- name: Generate tags for ultra-lite
id: meta-lite
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
with:
images: |
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
@@ -237,7 +214,7 @@ jobs:
- name: Build and push Unified Dockerfile (ultra-lite variant)
id: build-push-lite
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-lite.outputs.tags != ''
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
@@ -253,7 +230,7 @@ jobs:
sbom: true
- name: Sign ultra-lite images
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-lite.outputs.digest != ''
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
env:
DIGEST: ${{ steps.build-push-lite.outputs.digest }}
TAGS: ${{ steps.meta-lite.outputs.tags }}
@@ -263,136 +240,3 @@ jobs:
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
done
# Standalone unoserver image — versioned independently via
# docker/unoserver/VERSION. master/V2-master: publish <version>+latest
# only when the version is new. main/testMain: republish :alpha only
# when the source hash differs from the published image's annotation.
- name: Read unoserver image version
id: unoserverVersion
if: env.RUN_UNOSERVER == 'true'
run: |
version=$(tr -d '[:space:]' < docker/unoserver/VERSION)
if [ -z "$version" ]; then
echo "docker/unoserver/VERSION is empty"; exit 1
fi
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "Unoserver image version (from file): ${version}"
- name: Compute unoserver image source hash
id: unoserverHash
if: env.RUN_UNOSERVER == 'true'
run: |
set -eu
hash=$(cat \
docker/unoserver/Dockerfile \
docker/unoserver/entrypoint.sh \
docker/unoserver/healthcheck.sh \
docker/unoserver/VERSION \
| sha256sum | cut -d' ' -f1)
echo "hash=${hash}" >> "$GITHUB_OUTPUT"
echo "Unoserver source hash: ${hash}"
- name: Decide whether to publish unoserver image
id: unoserverDecision
if: env.RUN_UNOSERVER == 'true'
env:
UNOSERVER_VERSION: ${{ steps.unoserverVersion.outputs.version }}
UNOSERVER_HASH: ${{ steps.unoserverHash.outputs.hash }}
UNOSERVER_IMAGE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-unoserver
UNOSERVER_HASH_ANNOTATION: org.stirlingpdf.unoserver-source-hash
FORCE_REBUILD: ${{ inputs.force_unoserver_rebuild }}
GH_REF: ${{ github.ref }}
EVENT_NAME: ${{ github.event_name }}
run: |
set -eu
mode="skip"
tags=""
read_published_hash() {
local ref="$1"
docker buildx imagetools inspect "$ref" --raw 2>/dev/null \
| jq -r --arg key "$UNOSERVER_HASH_ANNOTATION" \
'.annotations[$key] // empty' \
2>/dev/null || true
}
# Manual dispatch from any branch routes to the :alpha publish path.
EFFECTIVE_REF="$GH_REF"
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
EFFECTIVE_REF="refs/heads/testMain"
fi
case "$EFFECTIVE_REF" in
refs/heads/master|refs/heads/V2-master)
if [ "${FORCE_REBUILD}" = "true" ]; then
echo "force_unoserver_rebuild=true — building stable regardless"
mode="stable"
tags="${UNOSERVER_IMAGE}:${UNOSERVER_VERSION},${UNOSERVER_IMAGE}:latest"
elif docker manifest inspect "${UNOSERVER_IMAGE}:${UNOSERVER_VERSION}" >/dev/null 2>&1; then
echo "stirling-unoserver:${UNOSERVER_VERSION} already on GHCR — skipping"
else
echo "stirling-unoserver:${UNOSERVER_VERSION} is new — will publish"
mode="stable"
tags="${UNOSERVER_IMAGE}:${UNOSERVER_VERSION},${UNOSERVER_IMAGE}:latest"
fi
;;
refs/heads/main|refs/heads/testMain)
published_hash=$(read_published_hash "${UNOSERVER_IMAGE}:alpha")
if [ "${FORCE_REBUILD}" = "true" ]; then
echo "force_unoserver_rebuild=true — rebuilding :alpha regardless"
mode="alpha"
tags="${UNOSERVER_IMAGE}:alpha"
elif [ -n "$published_hash" ] && [ "$published_hash" = "$UNOSERVER_HASH" ]; then
echo "Published :alpha source hash matches (${published_hash}) — skipping"
else
if [ -z "$published_hash" ]; then
echo ":alpha has no source-hash annotation (first publish or pre-tracking image) — will publish"
else
echo "Source hash changed (was ${published_hash}, now ${UNOSERVER_HASH}) — will publish"
fi
mode="alpha"
tags="${UNOSERVER_IMAGE}:alpha"
fi
;;
*)
echo "Branch ${GH_REF} does not publish unoserver image"
;;
esac
echo "mode=${mode}" >> "$GITHUB_OUTPUT"
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
- name: Build and push unoserver image
id: build-push-unoserver
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode != 'skip'
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./docker/unoserver/Dockerfile
push: true
cache-from: type=gha,scope=stirling-unoserver
cache-to: type=gha,mode=max,scope=stirling-unoserver
tags: ${{ steps.unoserverDecision.outputs.tags }}
# Manifest annotation read by the decision step above to detect drift.
annotations: |
index:org.stirlingpdf.unoserver-source-hash=${{ steps.unoserverHash.outputs.hash }}
platforms: linux/amd64,linux/arm64/v8
provenance: true
sbom: true
- name: Sign unoserver image
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode == 'stable'
env:
DIGEST: ${{ steps.build-push-unoserver.outputs.digest }}
TAGS: ${{ steps.unoserverDecision.outputs.tags }}
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
run: |
if [ -n "$COSIGN_PRIVATE_KEY" ]; then
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
done
else
echo "Warning: COSIGN_PRIVATE_KEY not set, skipping unoserver image signing"
fi
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
packages: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+2 -2
View File
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -75,6 +75,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v3.29.5
uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v3.29.5
with:
sarif_file: results.sarif
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+4 -4
View File
@@ -37,7 +37,7 @@ jobs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -86,7 +86,7 @@ jobs:
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -557,7 +557,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -648,7 +648,7 @@ jobs:
if: always()
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
+2 -57
View File
@@ -48,7 +48,7 @@ jobs:
cache-scope: stirling-pdf-fat
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -80,7 +80,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: |
~/.gradle/wrapper
@@ -160,58 +160,3 @@ jobs:
build/reports/problems/
retention-days: 3
if-no-files-found: warn
test-build-unoserver-image:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Checkout Repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build docker/unoserver/Dockerfile
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./docker/unoserver/Dockerfile
push: false
load: true
cache-from: type=gha,scope=stirling-unoserver
cache-to: type=gha,mode=max,scope=stirling-unoserver
platforms: linux/amd64
tags: stirling-unoserver:pr-test
provenance: false
sbom: false
- name: Smoke test the built image
run: |
set -eu
docker run -d --name unoserver-smoke \
-e UNOSERVER_RECYCLE_INTERVAL_SECONDS=0 \
stirling-unoserver:pr-test
deadline=$((SECONDS + 60))
while [ $SECONDS -lt $deadline ]; do
status=$(docker inspect -f '{{.State.Health.Status}}' unoserver-smoke 2>/dev/null || echo "starting")
if [ "$status" = "healthy" ]; then
echo "unoserver became healthy"
docker logs unoserver-smoke | tail -30
docker rm -f unoserver-smoke
exit 0
fi
sleep 3
done
echo "unoserver did not become healthy in time"
docker logs unoserver-smoke || true
docker rm -f unoserver-smoke || true
exit 1
+4 -4
View File
@@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -131,7 +131,7 @@ jobs:
frontend: ${{ steps.changes.outputs.frontend }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -150,7 +150,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
@@ -186,7 +186,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
-1
View File
@@ -32,7 +32,6 @@ exampleYmlFiles/stirling/
/testing/cucumber/junit/
/testing/cucumber/report.html
/testing/.failed_tests
/.test-state/
SwaggerDoc.json
# Runtime storage for uploaded files and user data (not Java source code)
+48 -21
View File
@@ -1,8 +1,5 @@
version: '3'
# All gradle invocations go through `bash gradlew` (the sh wrapper) so they work
# uniformly on Linux/macOS (native bash) and Windows-with-Git-Bash.
tasks:
dev:
desc: "Start backend dev server"
@@ -13,43 +10,58 @@ tasks:
env:
SERVER_PORT: '{{.PORT}}'
cmds:
- '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}bash gradlew :stirling-pdf:bootRun'
dev:bundled:
desc: "Clean + bootRun with frontend bundled into the backend (single :8080 server)"
ignore_error: true
cmds:
- bash gradlew clean bootRun -PbuildWithFrontend=true
- cmd: '{{if .AIENGINE_URL}}set "AIENGINE_URL={{.AIENGINE_URL}}" && set "AIENGINE_ENABLED=true" && {{end}}cmd /c gradlew.bat :stirling-pdf:bootRun'
platforms: [windows]
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}./gradlew :stirling-pdf:bootRun'
platforms: [linux, darwin]
build:
desc: "Full backend build"
cmds:
- bash gradlew clean build
- cmd: cmd /c gradlew.bat clean build
platforms: [windows]
- cmd: ./gradlew clean build
platforms: [linux, darwin]
build:fast:
desc: "Build without tests"
cmds:
- bash gradlew clean build -x test
- cmd: cmd /c gradlew.bat clean build -x test
platforms: [windows]
- cmd: ./gradlew clean build -x test
platforms: [linux, darwin]
build:ci:
desc: "Build for CI (formatting checked separately)"
cmds:
- bash gradlew build -PnoSpotless
- cmd: cmd /c gradlew.bat build -PnoSpotless
platforms: [windows]
- cmd: ./gradlew build -PnoSpotless
platforms: [linux, darwin]
test:
desc: "Run backend tests"
cmds:
- bash gradlew test
- cmd: cmd /c gradlew.bat test
platforms: [windows]
- cmd: ./gradlew test
platforms: [linux, darwin]
format:
desc: "Auto-fix code formatting"
cmds:
- bash gradlew spotlessApply
- cmd: cmd /c gradlew.bat spotlessApply
platforms: [windows]
- cmd: ./gradlew spotlessApply
platforms: [linux, darwin]
format:check:
desc: "Check code formatting"
cmds:
- bash gradlew spotlessCheck
- cmd: cmd /c gradlew.bat spotlessCheck
platforms: [windows]
- cmd: ./gradlew spotlessCheck
platforms: [linux, darwin]
fix:
desc: "Auto-fix backend"
@@ -59,7 +71,10 @@ tasks:
swagger:
desc: "Generate OpenAPI docs"
cmds:
- bash gradlew :stirling-pdf:copySwaggerDoc
- cmd: cmd /c gradlew.bat :stirling-pdf:copySwaggerDoc
platforms: [windows]
- cmd: ./gradlew :stirling-pdf:copySwaggerDoc
platforms: [linux, darwin]
sources:
- app/core/src/main/java/**/*.java
- app/proprietary/src/main/java/**/*.java
@@ -77,19 +92,31 @@ tasks:
desc: "Print project version"
silent: true
cmds:
- bash gradlew printVersion --quiet | tail -1
- cmd: cmd /c gradlew.bat printVersion --quiet | tail -1
platforms: [windows]
- cmd: ./gradlew printVersion --quiet | tail -1
platforms: [linux, darwin]
licenses:check:
desc: "Check dependency licenses"
cmds:
- bash gradlew checkLicense --no-parallel
- cmd: cmd /c gradlew.bat checkLicense --no-parallel
platforms: [windows]
- cmd: ./gradlew checkLicense --no-parallel
platforms: [linux, darwin]
licenses:generate:
desc: "Check and generate dependency license report"
cmds:
- bash gradlew checkLicense generateLicenseReport --no-parallel
- cmd: cmd /c gradlew.bat checkLicense generateLicenseReport --no-parallel
platforms: [windows]
- cmd: ./gradlew checkLicense generateLicenseReport --no-parallel
platforms: [linux, darwin]
clean:
desc: "Clean build artifacts"
cmds:
- bash gradlew clean
- cmd: cmd /c gradlew.bat clean
platforms: [windows]
- cmd: ./gradlew clean
platforms: [linux, darwin]
+6 -1
View File
@@ -1,7 +1,12 @@
version: '3'
vars:
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported"
# Single source of truth for the jlink module list (also read by
# frontend/scripts/build-universal-mac-jre.sh during CI). The desktop
# taskfile runs with dir: frontend (see Taskfile.yml), so this path
# is relative to that directory.
JLINK_MODULES:
sh: tr -d '\n' < src-tauri/jlink-modules.txt
tasks:
prepare:
-6
View File
@@ -55,9 +55,3 @@ tasks:
desc: "Tail docker compose logs"
cmds:
- docker compose -f {{.COMPOSE_DIR}}/docker-compose.yml logs -f
test:
desc: "Run full Docker integration test suite (builds all variants and tests them)"
ignore_error: true
cmds:
- bash testing/test.sh {{.CLI_ARGS}}
-204
View File
@@ -1,204 +0,0 @@
version: '3'
tasks:
install:
desc: "Install Playwright browsers"
dir: frontend
deps: [ ':frontend:install' ]
cmds:
- npx playwright install {{.CLI_ARGS}} --with-deps
stubbed:
desc: "Run stubbed E2E tests"
dir: frontend
deps: [ ':frontend:prepare' ]
cmds:
- npx playwright test --project=stubbed {{.CLI_ARGS}}
live:
desc: "Run live E2E tests"
summary: |
Auto-spawns a Spring Boot backend with isolated state under
.test-state/playwright/ (purged on every run) so the suite doesn't
touch your local DB, settings.yml, backups or customFiles. The
backend is stopped automatically when the runner exits.
Pass extra Playwright flags via -- :
task e2e:live -- --headed --grep merge
deps:
- live:backend
- live:runner
live:backend:
internal: true
ignore_error: true
vars:
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
env:
STIRLING_BASE_PATH: '{{.BASE_DIR}}'
# Suppress the analytics opt-in modal that fires on first admin login.
# It renders a Mantine overlay that intercepts pointer events and blocks
# FirstLoginSlide from rendering, so the bootstrap spec times out waiting
# for the password-change prompt.
SYSTEM_ENABLEANALYTICS: "false"
# NOTE: SECURITY_INITIALLOGIN_USERNAME/PASSWORD are intentionally NOT set.
# The live-setup project's bootstrap spec performs the real first-login
# flow against the backend's default admin/stirling user, exercising the
# forced-password-change UI and leaving the DB at admin/adminadmin for
# the rest of the live suite. This is both real coverage of the first-
# login flow and a stronger seed than env-var-driven user creation.
cmds:
# Wrapped in `bash -c` because Task's embedded shell (mvdan/sh) doesn't
# support `&` + `$!` + `wait` reliably (gradle was never actually
# backgrounded, so `$!` recorded a stale PID and the runner saw an
# immediate "backend died"). Real bash backgrounds gradle properly and
# gives us a stable PID to clean up.
- |
bash -c '
set -e
rm -rf "{{.BASE_DIR}}"
mkdir -p "{{.BASE_DIR}}"
# Background gradle and record its PID so the runner can clean up
# the exact process tree (wrapper + forked Spring Boot JVM) without
# resorting to fuzzy `pkill -f` patterns. `wait` keeps this script
# alive for the lifetime of gradle so Task'"'"'s parallel deps stay
# synchronised.
bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 &
GRADLE_PID=$!
echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid"
wait $GRADLE_PID
'
live:runner:
internal: true
deps: [ ':frontend:prepare' ]
dir: frontend
vars:
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
cmds:
# Wrapped in `bash -c` because Task's embedded shell (mvdan/sh) only
# supports `EXIT`/`ERR` in `trap`, not `INT`/`TERM`. CLI_ARGS are
# passed positionally so quoted args like --grep "foo bar" survive.
- |
bash -c '
set +e
# bootRun forks a separate Spring Boot JVM as a child of the gradle
# wrapper; killing only the wrapper leaves that JVM orphaned holding
# :8080. Walk the recorded PID'"'"'s process tree via pgrep so we kill
# every descendant, then lsof as a last-resort backstop.
kill_tree() {
local pid=$1
for child in $(pgrep -P "$pid" 2>/dev/null); do
kill_tree "$child"
done
kill "$pid" 2>/dev/null || true
}
cleanup() {
if [ -f "{{.BASE_DIR}}/backend.pid" ]; then
echo "Stopping backend..."
kill_tree "$(cat "{{.BASE_DIR}}/backend.pid")"
sleep 1
lsof -ti:8080 2>/dev/null | xargs kill 2>/dev/null || true
echo "Backend stopped"
fi
}
# Trap so cleanup runs on every exit path: tests pass, tests fail,
# backend never starts, user Ctrl-Cs. Without this the gradle JVM
# is left orphaned holding :8080 whenever the wait loop times out.
trap cleanup EXIT INT TERM
echo "Waiting for backend on :8080 (up to 10min; first compile can be slow)..."
WAITED=0
while [ $WAITED -lt 600 ]; do
if curl -sf http://localhost:8080/api/v1/info/status 2>/dev/null | grep -q UP; then
echo "Backend ready, starting Playwright"
break
fi
# Bail early if the backend already died: no point waiting 10min.
if [ -f "{{.BASE_DIR}}/backend.pid" ]; then
BACKEND_PID=$(cat "{{.BASE_DIR}}/backend.pid")
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
echo "Backend process exited before becoming ready; last 100 log lines:"
tail -100 "{{.BASE_DIR}}/backend.log" 2>/dev/null || true
exit 1
fi
fi
sleep 2
WAITED=$((WAITED + 2))
done
if [ $WAITED -ge 600 ]; then
echo "Backend did not become ready within 10min, aborting"
exit 1
fi
npx playwright test --project=live "$@"
' bash {{.CLI_ARGS}}
enterprise:
desc: "Run enterprise E2E tests"
summary: |
Requires an already-running keycloak compose stack on :8080. Bring one
up first with:
task e2e:oauth:up
task e2e:saml:up
dir: frontend
deps: [ ':frontend:prepare' ]
cmds:
- npx playwright test --project=enterprise {{.CLI_ARGS}}
cross-browser:
desc: "Run stubbed E2E tests on Chromium, Firefox, and WebKit"
dir: frontend
deps: [ ':frontend:prepare' ]
cmds:
- npx playwright test --project=stubbed --project=stubbed-firefox --project=stubbed-webkit {{.CLI_ARGS}}
check:
desc: "Run all E2E tests not requiring Docker"
cmds:
- task: stubbed
- task: live
check:all:
desc: "Run all E2E tests"
summary: |
Includes the enterprise project, which requires a keycloak compose
stack on :8080. Bring one up first with:
task e2e:oauth:up
task e2e:saml:up
cmds:
- task: stubbed
- task: live
- task: enterprise
oauth:up:
desc: "Start the OAuth keycloak test environment"
summary: |
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
task e2e:oauth:up LICENSE_KEY=abc123
Pass extra flags via -- :
task e2e:oauth:up -- --auto --nobuild
ignore_error: true
cmds:
- bash testing/compose/start-oauth-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
oauth:down:
desc: "Stop the OAuth keycloak test environment"
cmds:
- docker compose -f testing/compose/docker-compose-keycloak-oauth.yml down -v
saml:up:
desc: "Start the SAML keycloak test environment"
summary: |
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
task e2e:saml:up LICENSE_KEY=abc123
Pass extra flags via -- :
task e2e:saml:up -- --auto --with-storage --nobuild --language sv-SE
ignore_error: true
cmds:
- bash testing/compose/start-saml-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
saml:down:
desc: "Stop the SAML keycloak test environment"
cmds:
- docker compose -f testing/compose/docker-compose-keycloak-saml.yml down -v
+1 -2
View File
@@ -86,9 +86,8 @@ tasks:
fix:
desc: "Auto-fix lint + format"
cmds:
- task: format # Can auto-fix some things that `lint:fix` can't like line length violations
- task: lint:fix
- task: format # Ensure that after lint fixing that the code is still formatted correctly
- task: format
check:
desc: "Full engine quality gate"
+12
View File
@@ -284,6 +284,18 @@ tasks:
cmds:
- npx vitest --coverage
test:e2e:
desc: "Run E2E tests"
deps: [prepare]
cmds:
- npx playwright test {{.CLI_ARGS}}
test:e2e:install:
desc: "Install E2E test browsers"
deps: [install]
cmds:
- npx playwright install {{.CLI_ARGS}} --with-deps
# ============================================================
# Code Generation
# ============================================================
+10 -10
View File
@@ -183,26 +183,26 @@ Use this pattern for desktop-specific or proprietary-specific features WITHOUT r
**Example - Desktop-specific footer:**
```typescript
// core/components/workbenchBar/WorkbenchBarFooterExtensions.tsx (stub)
interface WorkbenchBarFooterExtensionsProps {
// core/components/rightRail/RightRailFooterExtensions.tsx (stub)
interface RightRailFooterExtensionsProps {
className?: string;
}
export function WorkbenchBarFooterExtensions(_props: WorkbenchBarFooterExtensionsProps) {
export function RightRailFooterExtensions(_props: RightRailFooterExtensionsProps) {
return null; // Stub - does nothing in web builds
}
```
```tsx
// desktop/components/workbenchBar/WorkbenchBarFooterExtensions.tsx (real implementation)
// desktop/components/rightRail/RightRailFooterExtensions.tsx (real implementation)
import { Box } from '@mantine/core';
import { BackendHealthIndicator } from '@app/components/BackendHealthIndicator';
interface WorkbenchBarFooterExtensionsProps {
interface RightRailFooterExtensionsProps {
className?: string;
}
export function WorkbenchBarFooterExtensions({ className }: WorkbenchBarFooterExtensionsProps) {
export function RightRailFooterExtensions({ className }: RightRailFooterExtensionsProps) {
return (
<Box className={className}>
<BackendHealthIndicator />
@@ -212,15 +212,15 @@ export function WorkbenchBarFooterExtensions({ className }: WorkbenchBarFooterEx
```
```tsx
// core/components/shared/WorkbenchBar.tsx (usage - works in ALL builds)
import { WorkbenchBarFooterExtensions } from '@app/components/workbenchBar/WorkbenchBarFooterExtensions';
// core/components/shared/RightRail.tsx (usage - works in ALL builds)
import { RightRailFooterExtensions } from '@app/components/rightRail/RightRailFooterExtensions';
export function WorkbenchBar() {
export function RightRail() {
return (
<div>
{/* In web builds: renders nothing (stub returns null) */}
{/* In desktop builds: renders BackendHealthIndicator */}
<WorkbenchBarFooterExtensions className="workbench-bar-footer" />
<RightRailFooterExtensions className="right-rail-footer" />
</div>
);
}
-3
View File
@@ -22,9 +22,6 @@ includes:
desktop:
taskfile: .taskfiles/desktop.yml
dir: frontend
e2e:
taskfile: .taskfiles/e2e.yml
dir: .
tasks:
# ============================================================
-8
View File
@@ -51,12 +51,4 @@ dependencies {
api 'org.simplejavamail:outlook-module:8.12.6' // MSG file support
api 'jakarta.mail:jakarta.mail-api:2.1.5'
runtimeOnly 'org.eclipse.angus:angus-mail:2.0.5'
// Tabula table extraction — used by the shared PDF parser and directly by downstream modules.
// api-scoped so downstream modules (core, proprietary) retain it on their compile classpath.
api ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
}
@@ -1,73 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
* LineAlignmentTableParser}. The first parser returning a result above {@link
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
*/
@Service
@Primary
@RequiredArgsConstructor
@Slf4j
public class CompositeTableParser implements TableParser {
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
private final TabulaTableParser tabulaParser;
private final LineAlignmentTableParser lineAlignmentParser;
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
// Step 1: Tabula lattice mode (ruled/bordered tables).
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
if (!latticeResults.isEmpty()) {
log.debug(
"Page {}: using Tabula lattice ({} table(s))",
rawPage.pageNumber(),
latticeResults.size());
return latticeResults;
}
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
// parseStream is not on the TableParser interface — this intentionally couples to the
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
List<TableFragment> streamResults =
filterConfident(tabulaParser.parseStream(document, rawPage));
if (!streamResults.isEmpty()) {
log.debug(
"Page {}: using Tabula stream ({} table(s))",
rawPage.pageNumber(),
streamResults.size());
return streamResults;
}
// Step 3: Geometry-based line-alignment fallback.
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
if (!lineResults.isEmpty()) {
log.debug(
"Page {}: using LineAlignment ({} table(s))",
rawPage.pageNumber(),
lineResults.size());
return lineResults;
}
return List.of();
}
private List<TableFragment> filterConfident(List<TableFragment> tables) {
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
}
}
@@ -1,528 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Fallback {@link TableParser} for borderless financial tables using text geometry.
*
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
* on column consistency and anchor density (confidence ceiling 0.85).
*/
@Service
@Slf4j
public class LineAlignmentTableParser implements TableParser {
/** Width in points of each column position bucket. */
static final float COLUMN_BUCKET_PT = 5f;
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
private static final int COLUMN_MATCH_BUCKETS = 2;
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
private static final float MAX_GAP_FACTOR = 2.5f;
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
static final int MIN_TABLE_ROWS = 3;
/** Minimum confirmed column positions to form a valid table. */
static final int MIN_COLUMNS = 2;
/**
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
* rows).
*/
private static final double COLUMN_MIN_FREQUENCY = 0.40;
/**
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
* percent, nil dashes.
*/
private static final Pattern NUMERIC =
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
/**
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
* column-gap logic).
*/
static final float ROW_MERGE_TOLERANCE_PT = 2f;
// ── public API ───────────────────────────────────────────────────────────────────────────────
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
List<RawLine> lines = rawPage.lines();
if (lines.size() < MIN_TABLE_ROWS) return List.of();
float modalSpacing = computeModalSpacing(lines);
List<TokenizedLine> tokenized =
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
List<Float> columnGrid = buildColumnGrid(anchors);
if (columnGrid.size() < MIN_COLUMNS) {
log.debug(
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
rawPage.pageNumber(),
MIN_COLUMNS);
return List.of();
}
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
List<TableFragment> results = new ArrayList<>();
for (int i = 0; i < groups.size(); i++) {
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
.ifPresent(results::add);
}
log.debug(
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
rawPage.pageNumber(),
results.size(),
anchors.size(),
columnGrid.size());
return results;
}
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
/**
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
* split by LineBuilder.
*/
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
if (tokenized.size() < 2) return tokenized;
List<TokenizedLine> result = new ArrayList<>();
int i = 0;
while (i < tokenized.size()) {
float baseY = tokenized.get(i).line().bounds().y();
int j = i + 1;
while (j < tokenized.size()
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
<= ROW_MERGE_TOLERANCE_PT) {
j++;
}
if (j == i + 1) {
result.add(tokenized.get(i));
} else {
result.add(mergeGroup(tokenized.subList(i, j)));
}
i = j;
}
return result;
}
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
List<TextFragment> mergedFragments =
group.stream()
.flatMap(tl -> tl.line().fragments().stream())
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds mergedBounds =
group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(group.get(0).line().bounds());
RawLine mergedLine =
new RawLine(
group.get(0).line().lineId(),
mergedFragments,
mergedBounds,
group.get(0).line().pageNumber());
return tokenize(mergedLine);
}
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
/**
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
* fragment.
*/
TokenizedLine tokenize(RawLine line) {
List<LineToken> tokens = new ArrayList<>();
for (TextFragment frag : line.fragments()) {
tokens.addAll(tokensFromFragment(frag));
}
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
return new TokenizedLine(line, tokens, numeric);
}
private List<LineToken> tokensFromFragment(TextFragment frag) {
String raw = frag.text();
if (raw == null || raw.isBlank()) return List.of();
float fragX = frag.bounds().x();
float fragWidth = frag.bounds().width();
int rawLen = raw.length();
List<LineToken> result = new ArrayList<>();
int offset = 0;
for (String part : raw.split("\\s+")) {
if (part.isEmpty()) {
offset++;
continue;
}
int idx = raw.indexOf(part, offset);
if (idx < 0) idx = offset;
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
float tokenRight =
rawLen > 0
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
: fragX + fragWidth;
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
offset = idx + part.length();
}
return result;
}
// ── column grid ──────────────────────────────────────────────────────────────────────────────
/**
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
*/
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
// bucket → set of line indices that contributed a numeric token to that bucket
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
for (int i = 0; i < anchors.size(); i++) {
for (LineToken t : anchors.get(i).numeric()) {
int bucket = bucket(t.right());
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
}
}
int minHits =
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
// Confirmed buckets → average right-edge for that bucket
TreeMap<Integer, Float> confirmed = new TreeMap<>();
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
// Count distinct lines
long distinctLines = entry.getValue().stream().distinct().count();
if (distinctLines >= minHits) {
double avg =
entry.getValue().stream()
.distinct() // weight each line equally regardless of token count
.mapToDouble(
lineIdx ->
avgRightEdgeForBucket(
anchors, lineIdx, entry.getKey()))
.average()
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
confirmed.put(entry.getKey(), (float) avg);
}
}
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
}
/**
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
*/
private double avgRightEdgeForBucket(
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
return anchors.get(lineIdx).numeric().stream()
.filter(t -> bucket(t.right()) == targetBucket)
.mapToDouble(LineToken::right)
.average()
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
}
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
/**
* Groups anchor lines into table candidates, including adjacent label rows; a gap &gt;
* MAX_GAP_FACTOR × modal spacing splits groups.
*/
private List<List<TokenizedLine>> groupRows(
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
List<List<TokenizedLine>> groups = new ArrayList<>();
List<TokenizedLine> current = new ArrayList<>();
for (int i = 0; i < all.size(); i++) {
TokenizedLine tl = all.get(i);
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
if (current.isEmpty()) {
if (fits) current.add(tl);
continue;
}
float gap =
tl.line().bounds().y()
- current.get(current.size() - 1).line().bounds().bottom();
if (gap > maxGap) {
groups.add(current);
current = new ArrayList<>();
if (fits) current.add(tl);
continue;
}
if (fits) {
current.add(tl);
} else if (!tl.line().text().isBlank()) {
// Include non-anchor lines (labels) only if they have text and are within
// proximity.
current.add(tl);
}
}
if (!current.isEmpty()) groups.add(current);
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
}
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
>= MIN_TABLE_ROWS;
}
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long matches =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) matches / tl.numeric().size() >= 0.60;
}
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long hits =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) hits / tl.numeric().size() < 0.60;
}
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
private Optional<TableFragment> buildFragment(
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
List<String> warnings = new ArrayList<>();
List<List<String>> rawRows = new ArrayList<>();
List<TableRow> rows = new ArrayList<>();
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
TokenizedLine tl = group.get(rowIdx);
List<String> rawRow = buildRawRow(tl, columnGrid);
rawRows.add(Collections.unmodifiableList(rawRow));
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
}
// Column count = 1 label column + confirmed numeric columns
int colCount = columnGrid.size() + 1;
Bounds bounds = computeGroupBounds(group);
float confidence = computeConfidence(group, columnGrid, warnings);
return Optional.of(
new TableFragment(
"tbl-la-p" + pageNumber + "-" + tableIndex,
pageNumber,
bounds,
List.of(),
Collections.unmodifiableList(rows),
Collections.unmodifiableList(rawRows),
colCount,
confidence,
Collections.unmodifiableList(warnings),
null));
}
/**
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
*/
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
String[] cells = new String[columnGrid.size() + 1];
Arrays.fill(cells, "");
// Separate label tokens (those not landing in any confirmed column) from column tokens.
List<String> labelParts = new ArrayList<>();
for (LineToken token : tl.all()) {
int col = nearestColumnIndex(token.right(), columnGrid);
if (col >= 0 && token.numeric()) {
int cellIdx = col + 1;
cells[cellIdx] =
cells[cellIdx].isEmpty()
? token.text()
: cells[cellIdx] + " " + token.text();
} else {
labelParts.add(token.text());
}
}
cells[0] = String.join(" ", labelParts).trim();
return Arrays.asList(cells);
}
private TableRow buildTableRow(
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
List<TableCell> cells = new ArrayList<>(rawRow.size());
// Label cell: use the line's full bounds as an approximation.
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
for (int col = 0; col < columnGrid.size(); col++) {
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
float right = columnGrid.get(col);
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
Bounds cellBounds =
new Bounds(
left,
tl.line().bounds().y(),
right - left,
tl.line().bounds().height());
cells.add(TableCell.of(col + 1, text, cellBounds));
}
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
}
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
/**
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
* &gt;30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
*/
private float computeConfidence(
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
float score = 0.70f;
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
long totalRows = group.size();
// More columns
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
score += extraCols * 0.05f;
// More anchor rows
if (anchorCount >= 5) score += 0.05f;
if (anchorCount >= 8) score += 0.05f;
// Inconsistent column matching
long inconsistent =
group.stream()
.filter(TokenizedLine::isAnchor)
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
.count();
if (inconsistent > anchorCount * 0.30) {
score -= 0.15f;
warnings.add(
"Column match inconsistent on "
+ inconsistent
+ "/"
+ anchorCount
+ " anchor rows");
}
// Label-heavy
long nonAnchor = totalRows - anchorCount;
if (nonAnchor > anchorCount) {
score -= 0.10f;
warnings.add(
"Non-anchor rows ("
+ nonAnchor
+ ") outnumber anchor rows ("
+ anchorCount
+ ")");
}
return Math.max(0f, Math.min(0.85f, score));
}
// ── utility ──────────────────────────────────────────────────────────────────────────────────
/**
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
* #COLUMN_MATCH_BUCKETS} buckets.
*/
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
int nearest = -1;
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
for (int i = 0; i < grid.size(); i++) {
float dist = Math.abs(rightEdge - grid.get(i));
if (dist < minDist) {
minDist = dist;
nearest = i;
}
}
return nearest;
}
private Bounds computeGroupBounds(List<TokenizedLine> group) {
return group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
}
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
private float computeModalSpacing(List<RawLine> lines) {
if (lines.size() < 2) return 0f;
Map<Float, Long> freq = new HashMap<>();
for (int i = 1; i < lines.size(); i++) {
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
}
return freq.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse(0f);
}
private static int bucket(float x) {
return Math.round(x / COLUMN_BUCKET_PT);
}
// ── private data types ───────────────────────────────────────────────────────────────────────
/** A word-level token with an approximate right-edge x-position. */
record LineToken(String text, float x, float right, boolean numeric) {}
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
boolean isAnchor() {
return numeric.size() >= 2;
}
}
}
@@ -1,139 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
*
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
*/
@Service
@Slf4j
public class LineBuilder {
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
private static final float MIN_BASELINE_TOLERANCE = 2f;
/**
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
* two-column gutters.
*/
static final float COLUMN_GAP_RATIO = 0.10f;
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
static final float COLUMN_GAP_MIN_PT = 40f;
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
if (fragments.isEmpty()) return List.of();
float effectiveWidth = inferEffectiveWidth(fragments);
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
log.debug(
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
pageNumber,
effectiveWidth,
columnGapThreshold);
// Sort top-to-bottom first, then left-to-right within the same baseline band.
List<TextFragment> sorted =
fragments.stream()
.sorted(
Comparator.comparingDouble(TextFragment::baseline)
.thenComparingDouble(f -> f.bounds().x()))
.toList();
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
List<RawLine> lines = new ArrayList<>(groups.size());
for (int i = 0; i < groups.size(); i++) {
List<TextFragment> group =
groups.get(i).stream()
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds lineBounds =
group.stream()
.map(TextFragment::bounds)
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
}
return lines;
}
private List<List<TextFragment>> groupByBaseline(
List<TextFragment> sorted, float columnGapThreshold) {
List<List<TextFragment>> groups = new ArrayList<>();
List<TextFragment> current = new ArrayList<>();
float currentBaseline = Float.NaN;
for (TextFragment fragment : sorted) {
if (current.isEmpty()) {
current.add(fragment);
currentBaseline = fragment.baseline();
continue;
}
float maxFontSize =
Math.max(
fragment.fontSize(),
(float)
current.stream()
.mapToDouble(TextFragment::fontSize)
.max()
.orElse(0));
float tolerance =
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
if (sameBaseline && !columnGap) {
current.add(fragment);
// Anchor to the weighted mean baseline so long lines stay stable.
currentBaseline =
(currentBaseline * (current.size() - 1) + fragment.baseline())
/ current.size();
} else {
groups.add(current);
current = new ArrayList<>();
current.add(fragment);
currentBaseline = fragment.baseline();
}
}
if (!current.isEmpty()) groups.add(current);
return groups;
}
/**
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
* threshold}.
*/
private static boolean hasColumnGap(
TextFragment next, List<TextFragment> group, float threshold) {
float lastRight = group.get(group.size() - 1).bounds().right();
return next.bounds().x() - lastRight > threshold;
}
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
private static float inferEffectiveWidth(List<TextFragment> fragments) {
double maxRight =
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
return (float) maxRight * 1.10f;
}
}
@@ -1,79 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
* {@link PDDocument} lifecycle.
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class PdfIngester {
private final LineBuilder lineBuilder;
private final TableParser tableParser;
public List<ParsedPage> parse(PDDocument document) throws IOException {
return parse(document, document.getNumberOfPages());
}
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
List<ParsedPage> pages = new ArrayList<>(pageCount);
long fragmentsMs = 0;
long tablesMs = 0;
long t0 = System.currentTimeMillis();
for (int p = 1; p <= pageCount; p++) {
long ft = System.currentTimeMillis();
List<TextFragment> fragments = extractFragments(document, p);
fragmentsMs += System.currentTimeMillis() - ft;
PDPage page = document.getPage(p - 1);
PDRectangle mediaBox = page.getMediaBox();
List<RawLine> lines = lineBuilder.build(fragments, p);
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
long tt = System.currentTimeMillis();
List<TableFragment> tables = tableParser.parse(document, rawPage);
tablesMs += System.currentTimeMillis() - tt;
log.debug(
"Page {}: {} fragments → {} lines, {} table(s)",
p,
fragments.size(),
lines.size(),
tables.size());
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
}
log.info(
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
pageCount,
System.currentTimeMillis() - t0,
fragmentsMs,
tablesMs);
return pages;
}
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
throws IOException {
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
stripper.getText(document);
return stripper.getFragments();
}
}
@@ -1,148 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
/**
* All PDF parser model types and the table-parser contract in one place.
*
* <p>Import as {@code import static stirling.software.SPDF.pdf.parser.PdfModels.*;} to use all
* nested types without qualification.
*/
public final class PdfModels {
private PdfModels() {}
// ── Geometry ──────────────────────────────────────────────────────────────
public record Bounds(float x, float y, float width, float height) {
public float right() {
return x + width;
}
public float bottom() {
return y + height;
}
public static Bounds merge(Bounds a, Bounds b) {
float x = Math.min(a.x, b.x);
float y = Math.min(a.y, b.y);
return new Bounds(
x, y, Math.max(a.right(), b.right()) - x, Math.max(a.bottom(), b.bottom()) - y);
}
}
// ── Text fragments and lines ──────────────────────────────────────────────
/**
* A contiguous run of text from a single PDF content-stream string operation. {@code baseline}
* is preserved separately from bounds for line-grouping — characters of different sizes on the
* same visual line share a baseline but differ in top Y.
*/
public record TextFragment(
String fragmentId,
String text,
Bounds bounds,
float baseline,
float fontSize,
String fontName,
boolean bold) {}
public record RawLine(
String lineId, List<TextFragment> fragments, Bounds bounds, int pageNumber) {
public String text() {
if (fragments.isEmpty()) return "";
StringBuilder sb = new StringBuilder();
TextFragment prev = null;
for (TextFragment f : fragments) {
if (prev != null) {
float gap = f.bounds().x() - prev.bounds().right();
float avgCharWidth = prev.bounds().width() / Math.max(prev.text().length(), 1);
if (gap > avgCharWidth * 0.5f) sb.append(' ');
}
sb.append(f.text());
prev = f;
}
return sb.toString();
}
public float dominantFontSize() {
return fragments.stream()
.collect(Collectors.groupingBy(TextFragment::fontSize, Collectors.counting()))
.entrySet()
.stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse(0f);
}
public boolean hasBold() {
return fragments.stream().anyMatch(TextFragment::bold);
}
}
public record RawPage(int pageNumber, float widthPt, float heightPt, List<RawLine> lines) {}
// ── Table model ───────────────────────────────────────────────────────────
/**
* A single cell within a {@link TableRow}. {@code colSpan} and {@code rowSpan} are always 1 in
* v1 — span detection is deferred but the fields keep the schema stable.
*/
public record TableCell(int colIndex, String text, Bounds bounds, int colSpan, int rowSpan) {
public static TableCell of(int colIndex, String text, Bounds bounds) {
return new TableCell(colIndex, text, bounds, 1, 1);
}
}
public record TableRow(int rowIndex, List<TableCell> cells) {}
/**
* A table as it appears on a single page. {@code headers} is empty in v1 — all rows are in
* {@code rows}. {@code rawRows} preserves exact Tabula text output for diagnostics. {@code
* confidence} is a heuristic score in [0.0, 1.0]. {@code continuedFromPage} is null in v1.
*/
public record TableFragment(
String tableId,
int pageNumber,
Bounds bounds,
List<TableRow> headers,
List<TableRow> rows,
List<List<String>> rawRows,
int columnCount,
float confidence,
List<String> warnings,
Integer continuedFromPage) {}
// ── Page output ───────────────────────────────────────────────────────────
public record ParsedPage(
int pageNumber,
float widthPt,
float heightPt,
List<TableFragment> tables,
List<RawLine> layoutLines) {}
// ── Parser contract ───────────────────────────────────────────────────────
/**
* Extracts tables from a single page of a PDF. The caller owns the document lifecycle;
* implementations must not close it.
*/
public interface TableParser {
/**
* @param document open PDF; must not be closed by the implementation
* @param rawPage page metadata and lines for the page to process
* @return zero or more table fragments found on the page, never null
*/
List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException;
}
}
@@ -1,256 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.RectangularTextContainer;
import technology.tabula.Table;
import technology.tabula.extractors.BasicExtractionAlgorithm;
import technology.tabula.extractors.ExtractionAlgorithm;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
/**
* Primary {@link TableParser} implementation using Tabula's lattice-mode extraction.
*
* <h3>Algorithm</h3>
*
* Uses {@link SpreadsheetExtractionAlgorithm} (lattice mode), which detects tables from ruled lines
* (horizontal and vertical PDF path operators). This is reliable for tables with visible borders.
* Borderless or whitespace-delimited tables will not be detected — that requires stream mode, which
* is deferred to a later iteration.
*
* <h3>Coordinate system</h3>
*
* Tabula normalises page coordinates so that (0,0) is the top-left corner and Y increases downward,
* matching the {@link Bounds} convention used throughout this parser. No coordinate transformation
* is needed when mapping Tabula cell bounds to {@link Bounds}.
*
* <h3>Known limitations</h3>
*
* <ul>
* <li>Borderless tables are not detected (use stream mode, deferred).
* <li>Colspan and rowspan are not detected; all cells report colSpan=1, rowSpan=1.
* <li>Header rows are not identified; all rows appear in {@code rows}, headers is empty.
* <li>Cross-page table linking is not performed; each page is independent.
* <li>Rotated tables (90°/270° pages) may produce incorrect bounds.
* </ul>
*/
@Service
@Slf4j
public class TabulaTableParser implements TableParser {
/** Lattice mode — reliable for tables with visible ruled borders. */
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
return parseWithAlgorithm(
document, rawPage, new SpreadsheetExtractionAlgorithm(), "lattice");
}
/**
* Convenience overload for callers that only have a page number, not a full {@link RawPage}.
*/
public List<TableFragment> parse(PDDocument document, int pageNumber) throws IOException {
return parse(document, new RawPage(pageNumber, 0f, 0f, List.of()));
}
/** Stream mode — whitespace-based column detection for borderless tables. */
public List<TableFragment> parseStream(PDDocument document, RawPage rawPage)
throws IOException {
return parseWithAlgorithm(document, rawPage, new BasicExtractionAlgorithm(), "stream");
}
private List<TableFragment> parseWithAlgorithm(
PDDocument document, RawPage rawPage, ExtractionAlgorithm algorithm, String modeName)
throws IOException {
int pageNumber = rawPage.pageNumber();
List<Table> tabulaTables;
try {
// Do NOT use try-with-resources: ObjectExtractor.close() closes the underlying
// PDDocument, which we don't own. The extractor holds no resources of its own.
ObjectExtractor extractor = new ObjectExtractor(document);
Page page = extractor.extract(pageNumber);
tabulaTables = new ArrayList<>(algorithm.extract(page));
} catch (Exception e) {
log.warn(
"Tabula {} extraction failed on page {}: {}",
modeName,
pageNumber,
e.getMessage());
return List.of();
}
if (tabulaTables.isEmpty()) {
log.debug("Page {}: no tables detected by Tabula ({})", pageNumber, modeName);
return List.of();
}
log.debug(
"Page {}: Tabula ({}) detected {} table(s)",
pageNumber,
modeName,
tabulaTables.size());
List<TableFragment> fragments = new ArrayList<>(tabulaTables.size());
for (int i = 0; i < tabulaTables.size(); i++) {
fragments.add(toFragment(tabulaTables.get(i), pageNumber, i));
}
return fragments;
}
// ── private helpers ──────────────────────────────────────────────────────────────────────────
private TableFragment toFragment(Table table, int pageNumber, int tableIndex) {
List<List<RectangularTextContainer>> tabulaRows = table.getRows();
List<String> warnings = new ArrayList<>();
List<List<String>> rawRows = buildRawRows(tabulaRows);
int colCount = inferColumnCount(rawRows, warnings);
List<TableRow> rows = buildRows(tabulaRows, colCount, warnings);
float confidence = computeConfidence(rawRows, colCount, warnings);
Bounds bounds = tableBounds(table);
String tableId = "tbl-p" + pageNumber + "-" + tableIndex;
if (!warnings.isEmpty()) {
log.warn("Page {}, table {}: {}", pageNumber, tableIndex, warnings);
}
return new TableFragment(
tableId,
pageNumber,
bounds,
List.of(), // headers: deferred to v2
rows,
rawRows,
colCount,
confidence,
Collections.unmodifiableList(warnings),
null); // continuedFromPage: deferred to v2
}
private List<List<String>> buildRawRows(List<List<RectangularTextContainer>> tabulaRows) {
List<List<String>> rawRows = new ArrayList<>(tabulaRows.size());
for (List<RectangularTextContainer> tabulaRow : tabulaRows) {
List<String> cells = new ArrayList<>(tabulaRow.size());
for (RectangularTextContainer cell : tabulaRow) {
cells.add(normaliseText(cell.getText()));
}
rawRows.add(Collections.unmodifiableList(cells));
}
return rawRows;
}
private List<TableRow> buildRows(
List<List<RectangularTextContainer>> tabulaRows, int colCount, List<String> warnings) {
List<TableRow> rows = new ArrayList<>(tabulaRows.size());
for (int rowIdx = 0; rowIdx < tabulaRows.size(); rowIdx++) {
List<RectangularTextContainer> tabulaRow = tabulaRows.get(rowIdx);
List<TableCell> cells = new ArrayList<>(tabulaRow.size());
for (int colIdx = 0; colIdx < tabulaRow.size(); colIdx++) {
RectangularTextContainer c = tabulaRow.get(colIdx);
Bounds cellBounds =
new Bounds(
(float) c.getX(),
(float) c.getY(),
(float) c.getWidth(),
(float) c.getHeight());
cells.add(TableCell.of(colIdx, normaliseText(c.getText()), cellBounds));
}
if (tabulaRow.size() != colCount) {
warnings.add(
"Row "
+ rowIdx
+ " has "
+ tabulaRow.size()
+ " cells; expected "
+ colCount);
}
rows.add(new TableRow(rowIdx, Collections.unmodifiableList(cells)));
}
return rows;
}
/**
* The canonical column count for a table is the size of the widest row. Tabula can produce
* uneven rows when a cell's ruling lines are partially missing.
*/
private int inferColumnCount(List<List<String>> rawRows, List<String> warnings) {
if (rawRows.isEmpty()) return 0;
int max = rawRows.stream().mapToInt(List::size).max().orElse(0);
int mode =
rawRows.stream()
.collect(
java.util.stream.Collectors.groupingBy(
List::size, java.util.stream.Collectors.counting()))
.entrySet()
.stream()
.max(java.util.Map.Entry.comparingByValue())
.map(java.util.Map.Entry::getKey)
.orElse(0);
if (max != mode) {
warnings.add("Inconsistent column count: modal=" + mode + " max=" + max);
}
return mode > 0 ? mode : max;
}
/**
* Heuristic confidence score in [0.0, 1.0].
*
* <ul>
* <li>Starts at 1.0.
* <li>-0.3 if only one column (single-column tables are usually not real tables).
* <li>-0.1 per row with an inconsistent column count, capped at -0.4.
* <li>-0.3 if the empty-cell ratio across all cells exceeds 80%.
* </ul>
*/
private float computeConfidence(
List<List<String>> rawRows, int colCount, List<String> warnings) {
if (rawRows.isEmpty() || colCount == 0) return 0f;
float score = 1.0f;
if (colCount == 1) score -= 0.3f;
long inconsistentRows = rawRows.stream().filter(r -> r.size() != colCount).count();
score -= Math.min(inconsistentRows * 0.1f, 0.4f);
long totalCells = rawRows.stream().mapToLong(List::size).sum();
long emptyCells =
rawRows.stream().flatMap(Collection::stream).filter(String::isBlank).count();
if (totalCells > 0 && (float) emptyCells / totalCells > 0.8f) {
score -= 0.3f;
}
return Math.max(0f, Math.min(1f, score));
}
private Bounds tableBounds(Table table) {
return new Bounds(
(float) table.getX(),
(float) table.getY(),
(float) table.getWidth(),
(float) table.getHeight());
}
private String normaliseText(String raw) {
if (raw == null) return "";
// Tabula wraps multi-line cell content with \r\n — collapse to a single space.
return raw.replace("\r\n", " ").replace("\n", " ").replace("\r", " ").trim();
}
}
@@ -1,113 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
/**
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
*
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
*/
class WordExtractingStripper extends PDFTextStripper {
private final int targetPage;
private final List<TextFragment> fragments = new ArrayList<>();
private int fragmentIndex = 0;
WordExtractingStripper(int pageNumber) throws IOException {
this.targetPage = pageNumber;
setStartPage(pageNumber);
setEndPage(pageNumber);
setSortByPosition(true);
}
@Override
protected void startPage(PDPage page) throws IOException {
super.startPage(page);
fragments.clear();
fragmentIndex = 0;
}
@Override
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
if (text == null || text.isBlank()) return;
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
// number as its own string operation, so this is the common case).
if (text.indexOf(' ') < 0) {
emitFragment(text, textPositions);
return;
}
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
if (textPositions.size() != text.length()) {
emitFragment(text, textPositions);
return;
}
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
int start = 0;
for (int i = 0; i <= text.length(); i++) {
if (i == text.length() || text.charAt(i) == ' ') {
if (start < i) {
emitFragment(text.substring(start, i), textPositions.subList(start, i));
}
start = i + 1;
}
}
}
private void emitFragment(String text, List<TextPosition> positions) {
if (positions.isEmpty()) return;
float minX = Float.MAX_VALUE;
float minY = Float.MAX_VALUE;
float maxRight = -Float.MAX_VALUE;
float maxBaseline = -Float.MAX_VALUE;
TextPosition first = null;
for (TextPosition tp : positions) {
if (tp == null) continue;
if (first == null) first = tp;
float x = tp.getX();
// getY() is the baseline; top of character = getY() - getHeight().
float top = tp.getY() - tp.getHeight();
float right = x + tp.getWidth();
float baseline = tp.getY();
minX = Math.min(minX, x);
minY = Math.min(minY, top);
maxRight = Math.max(maxRight, right);
maxBaseline = Math.max(maxBaseline, baseline);
}
if (first == null) return;
PDFont font = first.getFont();
String fontName = font != null ? font.getName() : "";
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
float fontSize = first.getHeight();
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
}
List<TextFragment> getFragments() {
return Collections.unmodifiableList(fragments);
}
}
@@ -59,18 +59,6 @@ public class InstallationPathConfig {
}
private static String initializeBasePath() {
// Allow tests / harnesses to redirect the entire state tree (configs,
// backups, customFiles, pipeline, logs) to an isolated location via
// -Dstirling.base-path=... or STIRLING_BASE_PATH=... so a Playwright
// run never touches a developer's working state.
String override = System.getProperty("stirling.base-path");
if (override == null || override.isBlank()) {
override = System.getenv("STIRLING_BASE_PATH");
}
if (override != null && !override.isBlank()) {
boolean hasTrailingSeparator = override.endsWith("/") || override.endsWith("\\");
return hasTrailingSeparator ? override : override + File.separator;
}
return "." + File.separator;
}
@@ -76,7 +76,6 @@ public class ApplicationProperties {
private ProcessExecutor processExecutor = new ProcessExecutor();
private PdfEditor pdfEditor = new PdfEditor();
private AiEngine aiEngine = new AiEngine();
private InternalApi internalApi = new InternalApi();
@Bean
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
@@ -247,19 +246,6 @@ public class ApplicationProperties {
private int longRunningTimeoutSeconds = 600;
}
/**
* HTTP timeouts for loopback calls to internal Stirling API endpoints, used by the AI workflow
* executor and the pipeline processor. A bounded read timeout prevents a hung tool (e.g. an
* infinite loop in a PDF processing service) from stalling the entire chat workflow forever.
* Tools that legitimately need longer than the read timeout should be invoked through the async
* job executor instead of synchronously.
*/
@Data
public static class InternalApi {
private int connectTimeoutSeconds = 10;
private int readTimeoutSeconds = 300;
}
@Data
public static class Legal {
private String termsAndConditions;
@@ -14,6 +14,7 @@ import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@EqualsAndHashCode
@Schema(description = "PDF file input - either upload a file or provide a server-side file ID")
public class PDFFile {
@Schema(
@@ -1,19 +0,0 @@
package stirling.software.common.model.api.general;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
@Data
@EqualsAndHashCode
public class EditTextOperation {
@Schema(description = "The literal text to find.", requiredMode = Schema.RequiredMode.REQUIRED)
private String find;
@Schema(
description = "The replacement text. May be empty to delete the matched text.",
requiredMode = Schema.RequiredMode.REQUIRED)
private String replace;
}
@@ -5,7 +5,6 @@ import java.io.UncheckedIOException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.Duration;
import java.util.regex.Pattern;
import org.springframework.beans.factory.annotation.Autowired;
@@ -13,18 +12,15 @@ import org.springframework.core.env.Environment;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Service;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.RestTemplate;
import jakarta.servlet.ServletContext;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.enumeration.Role;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@@ -54,29 +50,16 @@ public class InternalApiClient {
private final UserServiceInterface userService;
private final TempFileManager tempFileManager;
private final Environment environment;
private final Duration readTimeout;
private final RestTemplate restTemplate;
public InternalApiClient(
ServletContext servletContext,
@Autowired(required = false) UserServiceInterface userService,
TempFileManager tempFileManager,
Environment environment,
ApplicationProperties applicationProperties) {
Environment environment) {
this.servletContext = servletContext;
this.userService = userService;
this.tempFileManager = tempFileManager;
this.environment = environment;
ApplicationProperties.InternalApi internalApi = applicationProperties.getInternalApi();
// A bounded read timeout is what protects the workflow when an internal tool hangs
// (e.g. an infinite loop in a PDF processing service). The connect timeout is short
// because this is a loopback call; if connecting takes longer than a few seconds the
// local server is itself unhealthy.
this.readTimeout = Duration.ofSeconds(internalApi.getReadTimeoutSeconds());
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setConnectTimeout(Duration.ofSeconds(internalApi.getConnectTimeoutSeconds()));
factory.setReadTimeout(readTimeout);
this.restTemplate = new RestTemplate(factory);
}
/**
@@ -91,6 +74,7 @@ public class InternalApiClient {
validateUrl(endpointPath);
String url = getBaseUrl() + endpointPath;
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
String apiKey = getApiKeyForUser();
if (apiKey != null && !apiKey.isEmpty()) {
@@ -100,38 +84,26 @@ public class InternalApiClient {
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
RequestCallback requestCallback = restTemplate.httpEntityCallback(entity, Resource.class);
try {
return restTemplate.execute(
url,
HttpMethod.POST,
requestCallback,
response -> {
try {
TempFile tempFile =
tempFileManager.createManagedTempFile("internal-api");
Files.copy(
response.getBody(),
tempFile.getPath(),
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
String filename = extractFilename(response.getHeaders());
TempFileResource resource = new TempFileResource(tempFile, filename);
return ResponseEntity.status(response.getStatusCode())
.headers(response.getHeaders())
.body(resource);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
} catch (ResourceAccessException e) {
// RestTemplate wraps low-level I/O failures in ResourceAccessException. Only the
// SocketTimeoutException-rooted case is a real timeout; other I/O failures (connection
// refused, DNS, etc.) propagate as-is so the upstream generic handler can describe
// them accurately.
if (e.getCause() instanceof java.net.SocketTimeoutException) {
throw new InternalApiTimeoutException(endpointPath, readTimeout, e);
}
throw e;
}
return restTemplate.execute(
url,
HttpMethod.POST,
requestCallback,
response -> {
try {
TempFile tempFile = tempFileManager.createManagedTempFile("internal-api");
Files.copy(
response.getBody(),
tempFile.getPath(),
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
String filename = extractFilename(response.getHeaders());
TempFileResource resource = new TempFileResource(tempFile, filename);
return ResponseEntity.status(response.getStatusCode())
.headers(response.getHeaders())
.body(resource);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
}
/**
@@ -1,36 +0,0 @@
package stirling.software.common.service;
import java.time.Duration;
/**
* Thrown when an internal Stirling tool invocation exceeds its configured read timeout or otherwise
* fails at the transport layer. Distinguishes a hung/timed-out tool from a tool that returned a
* non-2xx HTTP response (which is reported via the response status itself), so callers can present
* a clear "the tool didn't respond in time" message to the user instead of a generic stack trace.
*/
public class InternalApiTimeoutException extends RuntimeException {
private final String endpointPath;
private final Duration readTimeout;
public InternalApiTimeoutException(String endpointPath, Duration readTimeout, Throwable cause) {
super(buildMessage(endpointPath, readTimeout, cause), cause);
this.endpointPath = endpointPath;
this.readTimeout = readTimeout;
}
public String getEndpointPath() {
return endpointPath;
}
public Duration getReadTimeout() {
return readTimeout;
}
private static String buildMessage(String endpointPath, Duration readTimeout, Throwable cause) {
String reason = cause != null && cause.getMessage() != null ? cause.getMessage() : "";
return String.format(
"Internal tool %s did not respond within %ds (%s)",
endpointPath, readTimeout.toSeconds(), reason);
}
}
@@ -2289,85 +2289,6 @@ public class FormUtils {
acroForm.getFields().add(field);
}
/** Drops AcroForm fields whose widgets are no longer on any page of {@code document}. */
public void pruneOrphanedFormFields(PDDocument document) {
if (document == null) {
return;
}
PDDocumentCatalog catalog = document.getDocumentCatalog();
if (catalog == null) {
return;
}
PDAcroForm form = catalog.getAcroForm(null);
if (form == null) {
return;
}
List<PDField> fields = form.getFields();
if (fields.isEmpty()) {
return;
}
Set<COSDictionary> liveWidgets = collectLiveWidgetDictionaries(document);
List<PDField> kept = pruneFieldList(fields, liveWidgets);
if (kept.isEmpty()) {
catalog.setAcroForm(null);
} else if (kept.size() != fields.size()) {
form.setFields(kept);
}
}
private Set<COSDictionary> collectLiveWidgetDictionaries(PDDocument document) {
Set<COSDictionary> live = new HashSet<>();
int pageCount = document.getNumberOfPages();
for (int i = 0; i < pageCount; i++) {
try {
for (PDAnnotation annotation : document.getPage(i).getAnnotations()) {
if (annotation instanceof PDAnnotationWidget) {
live.add(annotation.getCOSObject());
}
}
} catch (IOException e) {
log.debug("Failed reading page {} annotations: {}", i, e.getMessage());
}
}
return live;
}
private List<PDField> pruneFieldList(List<PDField> fields, Set<COSDictionary> liveWidgets) {
List<PDField> kept = new ArrayList<>(fields.size());
for (PDField field : fields) {
if (field instanceof PDNonTerminalField nonTerminal) {
List<PDField> children = nonTerminal.getChildren();
List<PDField> remaining = pruneFieldList(children, liveWidgets);
if (remaining.isEmpty()) {
continue;
}
if (remaining.size() != children.size()) {
nonTerminal.setChildren(remaining);
}
kept.add(nonTerminal);
} else if (field instanceof PDTerminalField terminal) {
List<PDAnnotationWidget> widgets = terminal.getWidgets();
List<PDAnnotationWidget> liveOnes = new ArrayList<>(widgets.size());
for (PDAnnotationWidget widget : widgets) {
if (liveWidgets.contains(widget.getCOSObject())) {
liveOnes.add(widget);
}
}
if (liveOnes.isEmpty()) {
continue;
}
if (liveOnes.size() != widgets.size()) {
terminal.setWidgets(liveOnes);
}
kept.add(terminal);
} else {
kept.add(field);
}
}
return kept;
}
// Delegation methods to GeneralFormCopyUtils for form field transformation
public boolean hasAnyRotatedPage(PDDocument document) {
return stirling.software.common.util.GeneralFormCopyUtils.hasAnyRotatedPage(document);
@@ -15,7 +15,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import io.github.pixee.security.BoundedLineReader;
@@ -203,15 +202,7 @@ public class ProcessExecutor {
boolean useSemaphore = true;
List<String> commandToRun = command;
if (shouldUseUnoServerPool(command)) {
try {
unoLease = unoServerPool.acquireEndpoint(timeoutDuration, TimeUnit.MINUTES);
} catch (TimeoutException e) {
throw new IOException(
"All unoserver endpoints busy; request timed out after "
+ timeoutDuration
+ " minutes",
e);
}
unoLease = unoServerPool.acquireEndpoint();
commandToRun = applyUnoServerEndpoint(command, unoLease.getEndpoint());
useSemaphore = false;
}
@@ -3,11 +3,8 @@ package stirling.software.common.util;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import stirling.software.common.model.ApplicationProperties;
@@ -45,27 +42,6 @@ public class UnoServerPool {
return new UnoServerLease(endpoints.get(index), index, this);
}
/** Fail-fast variant; non-positive timeout falls back to unbounded acquire. */
public UnoServerLease acquireEndpoint(long timeout, TimeUnit unit)
throws InterruptedException, TimeoutException {
if (endpoints.isEmpty()) {
return new UnoServerLease(defaultEndpoint(), null, this);
}
if (timeout <= 0) {
return acquireEndpoint();
}
Integer index = availableIndices.poll(timeout, unit);
if (index == null) {
throw new TimeoutException(
"Timed out waiting for a free unoserver endpoint after "
+ timeout
+ " "
+ unit.name().toLowerCase(Locale.ROOT));
}
return new UnoServerLease(endpoints.get(index), index, this);
}
private void releaseEndpoint(Integer index) {
if (index != null) {
availableIndices.offer(index);
@@ -6,18 +6,15 @@ import java.util.List;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.api.security.RedactionArea;
import tools.jackson.core.type.TypeReference;
import tools.jackson.databind.DeserializationFeature;
import tools.jackson.databind.JavaType;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.json.JsonMapper;
/**
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
*/
@Slf4j
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
private final ObjectMapper objectMapper =
JsonMapper.builder()
@@ -25,12 +22,6 @@ public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.build();
private final Class<T> elementType;
public StringToArrayListPropertyEditor(Class<T> elementType) {
this.elementType = elementType;
}
@Override
public void setAsText(String text) throws IllegalArgumentException {
if (text == null || text.trim().isEmpty()) {
@@ -38,11 +29,8 @@ public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
return;
}
try {
JavaType listType =
objectMapper
.getTypeFactory()
.constructCollectionType(ArrayList.class, elementType);
List<T> list = objectMapper.readValue(text, listType);
TypeReference<ArrayList<RedactionArea>> typeRef = new TypeReference<>() {};
List<RedactionArea> list = objectMapper.readValue(text, typeRef);
setValue(list);
} catch (Exception e) {
log.error("Exception while converting {}", e);
@@ -1,153 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static org.assertj.core.api.Assertions.assertThat;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.List;
import org.junit.jupiter.api.Test;
/**
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
* column-grid construction.
*/
class LineAlignmentTableParserTest {
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
@Test
void mergeCoincidentLines_singleLine_unchanged() {
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
}
@Test
void mergeCoincidentLines_distinctYLines_unchanged() {
// Two lines at different y positions — must NOT be merged.
var lines =
List.of(
tokenized(rawLine(10f, 100f, "Revenue")),
tokenized(rawLine(10f, 115f, "Cost")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
}
@Test
void mergeCoincidentLines_sameY_merged() {
// Simulates a financial-table row split by LineBuilder at the column gap:
// label fragment at x=72 → "Revenue"
// value fragment at x=350 → "1,234"
// Both have y=100. After merge they should form one TokenizedLine.
var label = rawLine(72f, 100f, "Revenue");
var value = rawLine(350f, 100f, "1,234");
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
assertThat(merged).hasSize(1);
// The merged line should contain tokens from both halves.
var tokens = merged.get(0).all();
assertThat(tokens.stream().map(t -> t.text()).toList())
.containsExactlyInAnyOrder("Revenue", "1,234");
}
@Test
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
var bounds = merged.get(0).line().bounds();
assertThat(bounds.x()).isEqualTo(72f);
assertThat(bounds.right()).isEqualTo(380f);
}
@Test
void mergeCoincidentLines_withinTolerance_merged() {
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 101.5f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(1);
}
@Test
void mergeCoincidentLines_beyondTolerance_notMerged() {
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 103.0f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_threeCoincident_allMerged() {
// Three fragments at the same y (e.g. wide financial table with two value columns).
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(300f, 100f, "1,234");
var c = rawLine(400f, 100f, "5,678");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).all()).hasSize(3);
}
@Test
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
// After merging, the combined line should be an anchor (≥2 numeric tokens).
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
// Merged → anchor with at least 2 numerics.
var label = rawLine(72f, 100f, "Revenue");
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).isAnchor()).isTrue();
}
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
private static RawLine rawLine(float x, float y, String text) {
float width = text.length() * 6f; // ~6pt per char — rough but consistent
float height = 12f;
Bounds bounds = new Bounds(x, y, width, height);
TextFragment fragment =
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
return new RawLine("ln-test", List.of(fragment), bounds, 1);
}
/**
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
* Used to simulate a values-only line with multiple numeric tokens.
*/
private static RawLine rawLineMultiWord(
float x, float y, String word1, float w1, String word2, float w2) {
float height = 12f;
Bounds b1 = new Bounds(x, y, w1, height);
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
}
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
return parser.tokenize(line);
}
}
@@ -22,13 +22,11 @@ import org.springframework.mock.env.MockEnvironment;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.ResponseExtractor;
import org.springframework.web.client.RestTemplate;
import jakarta.servlet.ServletContext;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@@ -44,19 +42,8 @@ class InternalApiClientTest {
@BeforeEach
void setUp() {
lenient().when(servletContext.getContextPath()).thenReturn("");
client = newClient();
}
/**
* Build a fresh client. Tests that use {@link org.mockito.Mockito#mockConstruction} to
* intercept {@link RestTemplate} must call this from inside their {@code mockConstruction}
* block, since the client now caches one RestTemplate per instance at construction time.
*/
private InternalApiClient newClient() {
MockEnvironment environment = new MockEnvironment().withProperty("server.port", "8080");
ApplicationProperties applicationProperties = new ApplicationProperties();
return new InternalApiClient(
servletContext, userService, tempFileManager, environment, applicationProperties);
client = new InternalApiClient(servletContext, userService, tempFileManager, environment);
}
@Test
@@ -88,10 +75,7 @@ class InternalApiClientTest {
.thenAnswer(inv -> fakeOkResponse(inv.getArgument(3)));
})) {
// Reconstruct the client so its cached RestTemplate is the mocked one.
InternalApiClient mockedClient = newClient();
ResponseEntity<Resource> response =
mockedClient.post("/api/v1/general/merge-pdfs", body);
ResponseEntity<Resource> response = client.post("/api/v1/general/merge-pdfs", body);
assertNotNull(response);
assertEquals(HttpStatus.OK, response.getStatusCode());
@@ -102,72 +86,6 @@ class InternalApiClientTest {
}
}
@Test
void postWrapsSocketTimeoutAsInternalApiTimeoutException() {
// Simulates the read-timeout case: RestTemplate wraps SocketTimeoutException in
// ResourceAccessException when the underlying socket times out. The client must repackage
// that into a typed timeout exception that carries the failing endpoint and the configured
// read timeout, so the workflow layer can surface a clean "tool didn't respond" message
// to the user.
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
body.add("fileInput", namedResource("input.pdf", "data"));
try (var ignored =
mockConstruction(
RestTemplate.class,
(rt, ctx) -> {
when(rt.httpEntityCallback(any(), eq(Resource.class)))
.thenAnswer(inv -> (RequestCallback) req -> {});
when(rt.execute(anyString(), eq(HttpMethod.POST), any(), any()))
.thenThrow(
new ResourceAccessException(
"I/O error on POST request: Read timed out",
new java.net.SocketTimeoutException(
"Read timed out")));
})) {
InternalApiClient mockedClient = newClient();
InternalApiTimeoutException thrown =
assertThrows(
InternalApiTimeoutException.class,
() -> mockedClient.post("/api/v1/general/merge-pdfs", body));
assertEquals("/api/v1/general/merge-pdfs", thrown.getEndpointPath());
assertNotNull(thrown.getReadTimeout());
assertTrue(thrown.getMessage().contains("/api/v1/general/merge-pdfs"));
}
}
@Test
void postRethrowsNonTimeoutResourceAccessExceptionAsIs() {
// ResourceAccessException covers more than just timeouts (e.g. connection refused, DNS
// failure). Only SocketTimeoutException-rooted failures are timeouts; everything else
// must propagate so the upstream generic handler can label it correctly instead of lying
// about a "tool didn't respond" timeout.
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
body.add("fileInput", namedResource("input.pdf", "data"));
try (var ignored =
mockConstruction(
RestTemplate.class,
(rt, ctx) -> {
when(rt.httpEntityCallback(any(), eq(Resource.class)))
.thenAnswer(inv -> (RequestCallback) req -> {});
when(rt.execute(anyString(), eq(HttpMethod.POST), any(), any()))
.thenThrow(
new ResourceAccessException(
"I/O error on POST request: Connection refused",
new java.net.ConnectException(
"Connection refused")));
})) {
InternalApiClient mockedClient = newClient();
assertThrows(
ResourceAccessException.class,
() -> mockedClient.post("/api/v1/general/merge-pdfs", body));
}
}
@Test
void postRejectsDisallowedPath() {
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
@@ -207,10 +125,8 @@ class InternalApiClientTest {
.thenAnswer(inv -> fakeOkResponse(inv.getArgument(3)));
})) {
// Reconstruct the client so its cached RestTemplate is the mocked one.
InternalApiClient mockedClient = newClient();
ResponseEntity<Resource> response =
mockedClient.post("/api/v1/ai/tools/pdf-comment-agent", body);
client.post("/api/v1/ai/tools/pdf-comment-agent", body);
assertNotNull(response);
assertEquals(HttpStatus.OK, response.getStatusCode());
@@ -1,229 +0,0 @@
package stirling.software.common.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationWidget;
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
import org.apache.pdfbox.pdmodel.interactive.form.PDNonTerminalField;
import org.apache.pdfbox.pdmodel.interactive.form.PDTextField;
import org.junit.jupiter.api.Test;
class FormUtilsPruneOrphanedFieldsTest {
@Test
void noAcroFormIsNoOp() throws IOException {
try (PDDocument document = new PDDocument()) {
document.addPage(new PDPage(PDRectangle.A4));
FormUtils.pruneOrphanedFormFields(document);
assertNull(document.getDocumentCatalog().getAcroForm(null));
}
}
@Test
void dropsFieldsWhoseWidgetsAreAllOnRemovedPages() throws IOException {
byte[] pdfBytes = buildPdfWithFieldPerPage(3);
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(2);
document.removePage(0);
FormUtils.pruneOrphanedFormFields(document);
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
List<String> remainingNames = new ArrayList<>();
for (PDField field : form.getFields()) {
remainingNames.add(field.getPartialName());
}
assertEquals(List.of("field_1"), remainingNames);
}
}
@Test
void dropsAcroFormEntirelyWhenNoFieldsSurvive() throws IOException {
byte[] pdfBytes = buildPdfWithFieldPerPage(2);
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(1);
document.removePage(0);
document.addPage(new PDPage(PDRectangle.A4));
FormUtils.pruneOrphanedFormFields(document);
assertNull(document.getDocumentCatalog().getAcroForm(null));
}
}
@Test
void keepsLiveWidgetsAndDropsOrphanWidgetsFromMultiWidgetField() throws IOException {
byte[] pdfBytes = buildPdfWithMultiWidgetField();
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(0);
FormUtils.pruneOrphanedFormFields(document);
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
assertEquals(1, form.getFields().size());
PDField field = form.getFields().get(0);
assertEquals("multi", field.getPartialName());
assertEquals(2, field.getWidgets().size(), "two widgets remain after one is dropped");
}
}
@Test
void survivesRoundTripWithoutOrphanPagesInOutput() throws IOException {
byte[] pdfBytes = buildPdfWithFieldPerPage(3);
byte[] writtenBytes;
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(2);
document.removePage(1);
FormUtils.pruneOrphanedFormFields(document);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
writtenBytes = out.toByteArray();
}
}
try (PDDocument reloaded = Loader.loadPDF(writtenBytes)) {
assertEquals(1, reloaded.getNumberOfPages());
PDAcroForm form = reloaded.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
assertEquals(1, form.getFields().size());
assertEquals("field_0", form.getFields().get(0).getPartialName());
}
}
@Test
void prunesNestedNonTerminalFields() throws IOException {
byte[] pdfBytes = buildPdfWithNestedFields();
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(1);
FormUtils.pruneOrphanedFormFields(document);
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
assertEquals(1, form.getFields().size());
PDField group = form.getFields().get(0);
assertEquals("group", group.getPartialName());
assertTrue(group instanceof PDNonTerminalField);
PDNonTerminalField nonTerminal = (PDNonTerminalField) group;
assertEquals(1, nonTerminal.getChildren().size());
assertEquals("kept", nonTerminal.getChildren().get(0).getPartialName());
}
}
private static byte[] buildPdfWithFieldPerPage(int pageCount) throws IOException {
try (PDDocument document = new PDDocument()) {
PDAcroForm acroForm = new PDAcroForm(document);
acroForm.setDefaultResources(new PDResources());
document.getDocumentCatalog().setAcroForm(acroForm);
for (int i = 0; i < pageCount; i++) {
PDPage page = new PDPage(PDRectangle.A4);
document.addPage(page);
PDTextField field = new PDTextField(acroForm);
field.setPartialName("field_" + i);
PDAnnotationWidget widget = new PDAnnotationWidget();
widget.setRectangle(new PDRectangle(50, 50, 100, 20));
widget.setPage(page);
field.setWidgets(List.of(widget));
acroForm.getFields().add(field);
page.getAnnotations().add(widget);
}
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
return out.toByteArray();
}
}
}
private static byte[] buildPdfWithMultiWidgetField() throws IOException {
try (PDDocument document = new PDDocument()) {
PDAcroForm acroForm = new PDAcroForm(document);
acroForm.setDefaultResources(new PDResources());
document.getDocumentCatalog().setAcroForm(acroForm);
List<PDAnnotationWidget> widgets = new ArrayList<>();
for (int i = 0; i < 3; i++) {
PDPage page = new PDPage(PDRectangle.A4);
document.addPage(page);
PDAnnotationWidget widget = new PDAnnotationWidget();
widget.setRectangle(new PDRectangle(50, 50, 100, 20));
widget.setPage(page);
page.getAnnotations().add(widget);
widgets.add(widget);
}
PDTextField field = new PDTextField(acroForm);
field.setPartialName("multi");
field.setWidgets(widgets);
acroForm.getFields().add(field);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
return out.toByteArray();
}
}
}
private static byte[] buildPdfWithNestedFields() throws IOException {
try (PDDocument document = new PDDocument()) {
PDAcroForm acroForm = new PDAcroForm(document);
acroForm.setDefaultResources(new PDResources());
document.getDocumentCatalog().setAcroForm(acroForm);
PDPage pageA = new PDPage(PDRectangle.A4);
PDPage pageB = new PDPage(PDRectangle.A4);
document.addPage(pageA);
document.addPage(pageB);
PDNonTerminalField group = new PDNonTerminalField(acroForm);
group.setPartialName("group");
PDTextField kept = new PDTextField(acroForm);
kept.setPartialName("kept");
PDAnnotationWidget keptWidget = new PDAnnotationWidget();
keptWidget.setRectangle(new PDRectangle(50, 50, 100, 20));
keptWidget.setPage(pageA);
kept.setWidgets(List.of(keptWidget));
pageA.getAnnotations().add(keptWidget);
PDTextField dropped = new PDTextField(acroForm);
dropped.setPartialName("dropped");
PDAnnotationWidget droppedWidget = new PDAnnotationWidget();
droppedWidget.setRectangle(new PDRectangle(50, 100, 100, 20));
droppedWidget.setPage(pageB);
dropped.setWidgets(List.of(droppedWidget));
pageB.getAnnotations().add(droppedWidget);
group.setChildren(List.of(kept, dropped));
acroForm.getFields().add(group);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
return out.toByteArray();
}
}
}
}
@@ -9,7 +9,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Test;
@@ -174,70 +173,6 @@ public class UnoServerPoolTest {
assertEquals(port1, port2, "Should reuse the same endpoint after release");
}
@Test
void testAcquireWithTimeoutFailsFast() throws InterruptedException {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
createEndpoints(1);
UnoServerPool pool = new UnoServerPool(endpoints);
UnoServerPool.UnoServerLease held = pool.acquireEndpoint();
long start = System.nanoTime();
assertThrows(
TimeoutException.class,
() -> pool.acquireEndpoint(100, TimeUnit.MILLISECONDS),
"Should time out when no endpoint available");
long elapsedMs = (System.nanoTime() - start) / 1_000_000;
assertTrue(
elapsedMs >= 100 && elapsedMs < 1000,
"Should fail-fast after roughly the timeout, got " + elapsedMs + "ms");
held.close();
}
@Test
void testAcquireWithTimeoutSucceedsWhenAvailable()
throws InterruptedException, TimeoutException {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
createEndpoints(1);
UnoServerPool pool = new UnoServerPool(endpoints);
try (UnoServerPool.UnoServerLease lease = pool.acquireEndpoint(1, TimeUnit.SECONDS)) {
assertNotNull(lease);
assertEquals(2003, lease.getEndpoint().getPort());
}
}
@Test
void testAcquireWithZeroTimeoutBlocksUnbounded() throws InterruptedException {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
createEndpoints(1);
UnoServerPool pool = new UnoServerPool(endpoints);
UnoServerPool.UnoServerLease held = pool.acquireEndpoint();
AtomicInteger acquired = new AtomicInteger(0);
Thread t =
Thread.ofVirtual()
.start(
() -> {
try {
UnoServerPool.UnoServerLease lease =
pool.acquireEndpoint(0, TimeUnit.MILLISECONDS);
acquired.incrementAndGet();
lease.close();
} catch (Exception e) {
fail("unexpected: " + e);
}
});
Thread.sleep(150);
assertEquals(0, acquired.get(), "Should still be blocked");
held.close();
t.join(1000);
assertEquals(1, acquired.get(), "Should acquire after release");
}
@Test
void testHostLocationAndProtocol() throws InterruptedException {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints = new ArrayList<>();
@@ -11,11 +11,11 @@ import stirling.software.common.model.api.security.RedactionArea;
class StringToArrayListPropertyEditorTest {
private StringToArrayListPropertyEditor<RedactionArea> editor;
private StringToArrayListPropertyEditor editor;
@BeforeEach
void setUp() {
editor = new StringToArrayListPropertyEditor<>(RedactionArea.class);
editor = new StringToArrayListPropertyEditor();
}
@Test
+7 -1
View File
@@ -65,7 +65,7 @@ dependencies {
exclude group: 'com.fasterxml.jackson.jaxrs'
exclude group: 'com.fasterxml.jackson.module', module: 'jackson-module-jaxb-annotations'
}
implementation 'commons-io:commons-io:2.22.0'
implementation 'commons-io:commons-io:2.21.0'
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
implementation "org.bouncycastle:bcpkix-jdk18on:$bouncycastleVersion"
implementation 'io.micrometer:micrometer-core'
@@ -87,6 +87,12 @@ dependencies {
implementation 'com.sun.xml.bind:jaxb-core:4.0.7'
implementation 'org.apache.poi:poi-ooxml:5.5.1'
// https://mvnrepository.com/artifact/technology.tabula/tabula
implementation ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
// CVE-2022-25647: Explicit gson 2.13.2 to prevent unsafe deserialization (tabula would pull 2.8.7)
implementation 'com.google.code.gson:gson:2.13.2'
implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4'
@@ -1,333 +0,0 @@
package stirling.software.SPDF.controller.api;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.springframework.core.io.Resource;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.WebDataBinder;
import org.springframework.web.bind.annotation.InitBinder;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.multipart.MultipartFile;
import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.config.swagger.StandardPdfResponse;
import stirling.software.SPDF.model.api.general.EditTextRequest;
import stirling.software.SPDF.model.json.PdfJsonDocument;
import stirling.software.SPDF.model.json.PdfJsonPage;
import stirling.software.SPDF.model.json.PdfJsonTextElement;
import stirling.software.SPDF.service.PdfJsonConversionService;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.model.api.general.EditTextOperation;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor;
/**
* Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the
* input PDF is parsed into the editable JSON model, find/replace operations are applied to the
* {@code text} field of each text element, and the mutated model is rebuilt into a PDF.
*
* <p>Matching joins all text elements on a page into a single string before searching, so find
* strings can span multiple visual runs (titles split per word, kerning-broken phrases, etc.).
*
* <p>For cross-element matches the entire replacement is written into the first matched element,
* any intermediate elements are emptied, and the suffix of the last matched element is preserved.
* This anchors the new text at the leftmost match position and lets the font lay it out as one run,
* which is more visually reliable than redistributing words to the original per-element X positions
* (those positions are calibrated to the original word widths and rarely match the replacement
* widths). Centered or tracked layouts will become left-aligned at the original first-word position
* when their text is replaced - the trade-off is accepted in favour of avoiding glyph overlaps and
* gaps from variable-width word distribution.
*/
@Slf4j
@GeneralApi
@RequiredArgsConstructor
public class EditTextController {
private static final Pattern FILE_EXTENSION_PATTERN = Pattern.compile("[.][^.]+$");
private final PdfJsonConversionService pdfJsonConversionService;
private final TempFileManager tempFileManager;
@InitBinder
public void initBinder(WebDataBinder binder) {
binder.registerCustomEditor(
List.class,
"edits",
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
}
@AutoJobPostMapping(consumes = "multipart/form-data", value = "/edit-text")
@StandardPdfResponse
@Operation(
summary = "Edit text in a PDF via find and replace",
description =
"Applies an ordered list of find/replace operations to the text in a PDF and"
+ " returns the edited PDF. Useful for find-and-replace, bulk renames"
+ " (e.g. updating a company name throughout a document), and copy"
+ " editing where the AI agent has identified specific replacements."
+ " Matching is performed against the joined text of each page, so"
+ " find strings can span multiple visual runs (titles split per word,"
+ " kerning-broken phrases). Cross-element matches are written as a"
+ " single replacement run anchored at the leftmost matched position;"
+ " centered or tracked text may shift left when its content changes."
+ " Input:PDF Output:PDF Type:SISO")
public ResponseEntity<Resource> editText(@ModelAttribute EditTextRequest request)
throws Exception {
MultipartFile inputFile = request.getFileInput();
if (inputFile == null) {
throw ExceptionUtils.createFileNullOrEmptyException();
}
List<EditTextOperation> edits = request.getEdits();
if (edits == null || edits.isEmpty()) {
throw ExceptionUtils.createIllegalArgumentException(
"error.editText.no.edits",
"No find/replace operations provided for text editing");
}
for (EditTextOperation edit : edits) {
if (edit == null || edit.getFind() == null || edit.getFind().isEmpty()) {
throw ExceptionUtils.createIllegalArgumentException(
"error.editText.empty.find", "Each edit must have a non-empty find string");
}
}
boolean wholeWordSearch = Boolean.TRUE.equals(request.getWholeWordSearch());
List<CompiledEdit> compiledEdits = compileEdits(edits, wholeWordSearch);
PdfJsonDocument document = pdfJsonConversionService.convertPdfToJsonDocument(inputFile);
Set<Integer> pageFilter = resolvePageFilter(request, document);
int modifiedSpans = applyEdits(document, compiledEdits, pageFilter);
log.info(
"edit-text: modified {} text span(s) using {} edit(s) on {} page(s)",
modifiedSpans,
compiledEdits.size(),
pageFilter == null
? document.getPages() == null ? 0 : document.getPages().size()
: pageFilter.size());
String docName = buildOutputFilename(inputFile);
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
try (OutputStream os = Files.newOutputStream(tempOut.getPath())) {
pdfJsonConversionService.convertJsonToPdf(document, os);
} catch (Exception e) {
tempOut.close();
throw e;
}
return WebResponseUtils.pdfFileToWebResponse(tempOut, docName);
}
private List<CompiledEdit> compileEdits(
List<EditTextOperation> edits, boolean wholeWordSearch) {
return edits.stream().map(edit -> compileEdit(edit, wholeWordSearch)).toList();
}
private CompiledEdit compileEdit(EditTextOperation edit, boolean wholeWordSearch) {
// Always treat the user-supplied find string as a literal: Pattern.quote escapes any
// regex metacharacters, so the constructed pattern can only ever do a literal match
// (optionally bounded by our own word-boundary anchors). This rules out catastrophic
// backtracking from a malicious find string.
String findRaw = edit.getFind();
String replacement = Objects.toString(edit.getReplace(), "");
String regex = Pattern.quote(findRaw);
if (wholeWordSearch) {
// Use lookarounds rather than \b so the bound works even when the find string starts
// or ends with a non-word character (e.g. "-foo" or "$id"). \b only fires at a
// word/non-word transition, which would never trigger for a find that starts with a
// non-word char and was preceded by another non-word char in the source text.
regex = "(?<!\\w)(?:" + regex + ")(?!\\w)";
}
Pattern pattern = Pattern.compile(regex);
String safeReplacement = Matcher.quoteReplacement(replacement);
return new CompiledEdit(pattern, safeReplacement);
}
private Set<Integer> resolvePageFilter(EditTextRequest request, PdfJsonDocument document) {
String pageNumbers = request.getPageNumbers();
int totalPages = document.getPages() == null ? 0 : document.getPages().size();
if (totalPages == 0) {
return Collections.emptySet();
}
if (pageNumbers == null || pageNumbers.isBlank() || "all".equalsIgnoreCase(pageNumbers)) {
return null;
}
List<Integer> pages = GeneralUtils.parsePageList(pageNumbers, totalPages, true);
return new HashSet<>(pages);
}
private int applyEdits(
PdfJsonDocument document, List<CompiledEdit> edits, Set<Integer> pageFilter) {
List<PdfJsonPage> pages = document.getPages();
if (pages == null) {
return 0;
}
int modifiedSpans = 0;
// The page filter is built from parsePageList against pages.size(), so it returns 1-based
// positional page numbers. Match against the same positional numbering here rather than
// the per-page pageNumber field, which can become non-sequential after split/merge round
// trips and would misalign the filter.
for (int i = 0; i < pages.size(); i++) {
int positionalPageNumber = i + 1;
if (pageFilter != null && !pageFilter.contains(positionalPageNumber)) {
continue;
}
modifiedSpans += applyEditsToPage(pages.get(i), edits);
}
return modifiedSpans;
}
private int applyEditsToPage(PdfJsonPage page, List<CompiledEdit> edits) {
List<PdfJsonTextElement> elements = page.getTextElements();
if (elements == null || elements.isEmpty()) {
return 0;
}
Set<Integer> modifiedIndices = new HashSet<>();
for (CompiledEdit edit : edits) {
applyEditToPage(elements, edit, modifiedIndices);
}
for (Integer index : modifiedIndices) {
// Char codes were captured for the original glyph sequence; clear them so the rebuild
// re-encodes from the new text via the font.
elements.get(index).setCharCodes(null);
}
return modifiedIndices.size();
}
/**
* Apply a single edit across the page by matching against the concatenation of all element
* texts, then writing the replacement back into the originating element(s).
*/
private void applyEditToPage(
List<PdfJsonTextElement> elements, CompiledEdit edit, Set<Integer> modifiedIndices) {
StringBuilder joined = new StringBuilder();
int[] starts = new int[elements.size()];
int[] ends = new int[elements.size()];
for (int i = 0; i < elements.size(); i++) {
starts[i] = joined.length();
String text = elements.get(i).getText();
if (text != null) {
joined.append(text);
}
ends[i] = joined.length();
}
Matcher matcher = edit.pattern().matcher(joined);
List<MatchSpan> spans = new ArrayList<>();
StringBuffer interpolation = new StringBuffer();
int previousAppendPosition = 0;
while (matcher.find()) {
if (matcher.start() == matcher.end()) {
// Skip zero-length matches (e.g. /a*/ on empty input) — they cannot be applied.
continue;
}
int sizeBefore = interpolation.length();
matcher.appendReplacement(interpolation, edit.replacement());
int prefixLength = matcher.start() - previousAppendPosition;
String actualReplacement =
interpolation.substring(sizeBefore + prefixLength, interpolation.length());
spans.add(new MatchSpan(matcher.start(), matcher.end(), actualReplacement));
previousAppendPosition = matcher.end();
}
// Apply right-to-left so earlier match positions stay valid as we mutate elements.
for (int i = spans.size() - 1; i >= 0; i--) {
MatchSpan span = spans.get(i);
int firstElement = findElementForCharIndex(starts, ends, span.start());
int lastElement = findElementForCharIndex(starts, ends, span.end() - 1);
if (firstElement < 0 || lastElement < 0) {
continue;
}
applyMatchToElements(
elements, starts, span, firstElement, lastElement, modifiedIndices);
}
}
/**
* Find the element whose text covers the character at {@code charIndex} in the joined string.
* Returns -1 if no element covers that index (which should not happen for valid match spans).
*/
private static int findElementForCharIndex(int[] starts, int[] ends, int charIndex) {
for (int i = 0; i < starts.length; i++) {
if (starts[i] <= charIndex && charIndex < ends[i]) {
return i;
}
}
return -1;
}
private static void applyMatchToElements(
List<PdfJsonTextElement> elements,
int[] starts,
MatchSpan span,
int firstElement,
int lastElement,
Set<Integer> modifiedIndices) {
if (firstElement == lastElement) {
PdfJsonTextElement element = elements.get(firstElement);
String text = nullToEmpty(element.getText());
int matchStartInElement = span.start() - starts[firstElement];
int matchEndInElement = span.end() - starts[firstElement];
element.setText(
text.substring(0, matchStartInElement)
+ span.replacement()
+ text.substring(matchEndInElement));
modifiedIndices.add(firstElement);
return;
}
// Cross-element match: write the whole replacement into the first matched element, empty
// any intermediate elements, and keep only the suffix of the last matched element. The
// JSON->PDF rebuild concatenates per-token text, so the font lays out the replacement as
// one continuous run anchored at the first element's X position.
String firstText = nullToEmpty(elements.get(firstElement).getText());
int firstSplit = span.start() - starts[firstElement];
elements.get(firstElement).setText(firstText.substring(0, firstSplit) + span.replacement());
modifiedIndices.add(firstElement);
for (int mid = firstElement + 1; mid < lastElement; mid++) {
elements.get(mid).setText("");
modifiedIndices.add(mid);
}
String lastText = nullToEmpty(elements.get(lastElement).getText());
int lastSplit = span.end() - starts[lastElement];
elements.get(lastElement).setText(lastText.substring(lastSplit));
modifiedIndices.add(lastElement);
}
private static String nullToEmpty(String value) {
return value != null ? value : "";
}
private String buildOutputFilename(MultipartFile inputFile) {
String originalName = inputFile.getOriginalFilename();
String baseName =
(originalName != null && !originalName.isBlank())
? FILE_EXTENSION_PATTERN
.matcher(Filenames.toSimpleFileName(originalName))
.replaceFirst("")
: "document";
return baseName + "_edited.pdf";
}
private record CompiledEdit(Pattern pattern, String replacement) {}
private record MatchSpan(int start, int end, String replacement) {}
}
@@ -185,40 +185,39 @@ public class MergeController {
return new String[0];
}
// Adds a table of contents to the merged document using filenames as chapter titles
private void addTableOfContents(PDDocument mergedDocument, MultipartFile[] files) {
// Create the document outline
// Reads page counts from on-disk source files in read-only mode. A failed read falls back to
// 1 so TOC generation still produces a usable (if slightly misaligned) outline.
private int[] collectPageCounts(File[] sourceFiles) {
int[] counts = new int[sourceFiles.length];
for (int i = 0; i < sourceFiles.length; i++) {
try (PDDocument doc = pdfDocumentFactory.load(sourceFiles[i], true)) {
counts[i] = doc.getNumberOfPages();
} catch (IOException e) {
ExceptionUtils.logException("page count for TOC", e);
counts[i] = 1;
}
}
return counts;
}
// Adds a table of contents to the merged document using filenames as chapter titles.
// Page counts are passed in so we don't re-open every source PDF just to count pages.
private void addTableOfContents(
PDDocument mergedDocument, MultipartFile[] files, int[] pageCounts) {
PDDocumentOutline outline = new PDDocumentOutline();
mergedDocument.getDocumentCatalog().setDocumentOutline(outline);
int pageIndex = 0; // Current page index in the merged document
// Iterate through the original files
for (MultipartFile file : files) {
// Get the filename without extension to use as bookmark title
String filename = file.getOriginalFilename();
String title = GeneralUtils.removeExtension(filename);
// Create an outline item for this file
int pageIndex = 0;
for (int i = 0; i < files.length; i++) {
String title = GeneralUtils.removeExtension(files[i].getOriginalFilename());
PDOutlineItem item = new PDOutlineItem();
item.setTitle(title);
// Set the destination to the first page of this file in the merged document
if (pageIndex < mergedDocument.getNumberOfPages()) {
PDPage page = mergedDocument.getPage(pageIndex);
item.setDestination(page);
item.setDestination(mergedDocument.getPage(pageIndex));
}
// Add the item to the outline
outline.addLast(item);
// Increment page index for the next file
try (PDDocument doc = pdfDocumentFactory.load(file)) {
pageIndex += doc.getNumberOfPages();
} catch (IOException e) {
ExceptionUtils.logException("document loading for TOC generation", e);
pageIndex++; // Increment by at least one if we can't determine page count
}
int count = pageCounts[i];
pageIndex += count > 0 ? count : 1;
}
}
@@ -288,6 +287,7 @@ public class MergeController {
boolean removeCertSign = Boolean.TRUE.equals(request.getRemoveCertSign());
boolean generateToc = request.isGenerateToc();
boolean preserveAccessibility = request.isPreserveAccessibility();
MultipartFile[] files = request.getFileInput();
if (files == null) {
@@ -306,31 +306,34 @@ public class MergeController {
request.getSortType())); // Sort files based on requested sort type
}
try (TempFile mt = new TempFile(tempFileManager, ".pdf")) {
// Hold the merge output until response streaming completes. We only close it on failure;
// on success ownership transfers to the response (deleted when Spring closes the stream).
TempFile mergeOutput = new TempFile(tempFileManager, ".pdf");
boolean keepMergeOutput = false;
try {
PDFMergerUtility mergerUtility = new PDFMergerUtility();
// OPTIMIZE_RESOURCES_MODE closes source documents progressively and skips
// structure-tree copying — drops PDF/UA tags but uses much less heap.
// PDFBOX_LEGACY_MODE preserves tags at the cost of higher peak heap.
mergerUtility.setDocumentMergeMode(
preserveAccessibility
? PDFMergerUtility.DocumentMergeMode.PDFBOX_LEGACY_MODE
: PDFMergerUtility.DocumentMergeMode.OPTIMIZE_RESOURCES_MODE);
long totalSize = 0;
List<Integer> invalidIndexes = new ArrayList<>();
File[] sourceFiles = new File[files.length];
for (int index = 0; index < files.length; index++) {
MultipartFile multipartFile = files[index];
totalSize += multipartFile.getSize();
File tempFile =
tempFileManager.convertMultipartFileToFile(
multipartFile); // Convert MultipartFile to File
filesToDelete.add(tempFile); // Add temp file to the list for later deletion
// Pre-validate each PDF so we can report which one(s) are broken
// Use the original MultipartFile to avoid deleting the tempFile during validation
try (PDDocument ignored = pdfDocumentFactory.load(multipartFile)) {
// OK
} catch (IOException e) {
ExceptionUtils.logException("PDF pre-validate", e);
invalidIndexes.add(index);
}
mergerUtility.addSource(tempFile); // Add source file to the merger utility
File tempFile = tempFileManager.convertMultipartFileToFile(multipartFile);
filesToDelete.add(tempFile);
sourceFiles[index] = tempFile;
mergerUtility.addSource(tempFile);
}
// Pre-validation is intentionally omitted: PDFMergerUtility surfaces corrupted inputs
// via PdfErrorUtils.isCorruptedPdfError below, and a separate validation pass would
// double-allocate PDDocument graphs and re-spool every source >10 MB to disk.
mergerUtility.setDestinationFileName(mt.getFile().getAbsolutePath());
mergerUtility.setDestinationFileName(mergeOutput.getFile().getAbsolutePath());
try {
mergerUtility.mergeDocuments(
@@ -339,39 +342,59 @@ public class MergeController {
} catch (IOException e) {
ExceptionUtils.logException("PDF merge", e);
if (PdfErrorUtils.isCorruptedPdfError(e)) {
// Identify which source file(s) are corrupt for operator diagnostics.
// Only runs on the failure path so the happy path stays fast.
List<String> badFiles = new ArrayList<>();
for (int i = 0; i < sourceFiles.length; i++) {
try (PDDocument ignored =
pdfDocumentFactory.load(sourceFiles[i], true)) {
// OK
} catch (IOException corruptInput) {
String name = files[i].getOriginalFilename();
badFiles.add(name != null ? name : ("index " + i));
}
}
if (!badFiles.isEmpty()) {
log.warn("Corrupted PDFs in merge input: {}", badFiles);
}
throw ExceptionUtils.createMultiplePdfCorruptedException(e);
}
throw e;
}
// Load the merged PDF document and operate on it inside try-with-resources
try (PDDocument mergedDocument = pdfDocumentFactory.load(mt.getFile())) {
// Remove signatures if removeCertSign is true
if (removeCertSign) {
PDDocumentCatalog catalog = mergedDocument.getDocumentCatalog();
PDAcroForm acroForm = catalog.getAcroForm();
if (acroForm != null) {
List<PDField> fieldsToRemove =
acroForm.getFields().stream()
.filter(PDSignatureField.class::isInstance)
.toList();
// Common case: caller wants neither cert-sign removal nor a TOC. Skip the
// load-and-resave round-trip entirely — the merged file on disk is the response.
// For 4000+ page jobs this avoids materialising the merged PDDocument in heap.
if (!removeCertSign && !generateToc) {
outputTempFile = mergeOutput;
keepMergeOutput = true;
} else {
// Page counts are needed only when generating a TOC. Read them from the already-
// on-disk source files in read-only mode (no metadata mutation, no extra spool).
int[] pageCounts = generateToc ? collectPageCounts(sourceFiles) : null;
if (!fieldsToRemove.isEmpty()) {
acroForm.flatten(
fieldsToRemove,
false); // Flatten the fields, effectively removing them
outputTempFile = new TempFile(tempFileManager, ".pdf");
try (PDDocument mergedDocument = pdfDocumentFactory.load(mergeOutput.getFile())) {
// Resource cache off for the modify pass — we never call getImage() here,
// and disabling it prevents PDFBox from caching XObjects when the page tree
// is iterated during outline insertion or AcroForm flattening.
mergedDocument.setResourceCache(null);
if (removeCertSign) {
PDDocumentCatalog catalog = mergedDocument.getDocumentCatalog();
PDAcroForm acroForm = catalog.getAcroForm();
if (acroForm != null) {
List<PDField> fieldsToRemove =
acroForm.getFields().stream()
.filter(PDSignatureField.class::isInstance)
.toList();
if (!fieldsToRemove.isEmpty()) {
acroForm.flatten(fieldsToRemove, false);
}
}
}
}
// Add table of contents if generateToc is true
if (generateToc && files.length > 0) {
addTableOfContents(mergedDocument, files);
}
// Save the modified document to a temporary file
outputTempFile = new TempFile(tempFileManager, ".pdf");
try {
if (generateToc && files.length > 0) {
addTableOfContents(mergedDocument, files, pageCounts);
}
mergedDocument.save(outputTempFile.getFile());
} catch (Exception e) {
outputTempFile.close();
@@ -380,7 +403,7 @@ public class MergeController {
}
}
} catch (Exception ex) {
if (outputTempFile != null) {
if (outputTempFile != null && outputTempFile != mergeOutput) {
outputTempFile.close();
}
if (ex instanceof IOException && PdfErrorUtils.isCorruptedPdfError((IOException) ex)) {
@@ -390,6 +413,9 @@ public class MergeController {
}
throw ex;
} finally {
if (!keepMergeOutput && outputTempFile != mergeOutput) {
mergeOutput.close();
}
for (File file : filesToDelete) {
tempFileManager.deleteTempFile(file); // Delete temporary files
}
@@ -6,11 +6,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -30,7 +27,6 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.FormUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
@@ -71,7 +67,6 @@ public class RearrangePagesPDFController {
int pageIndex = pagesToRemove.get(i);
document.removePage(pageIndex);
}
FormUtils.pruneOrphanedFormFields(document);
return WebResponseUtils.pdfDocToWebResponse(
document,
GeneralUtils.generateFilename(
@@ -270,17 +265,6 @@ public class RearrangePagesPDFController {
rearrangedDocument.addPage(page);
}
PDDocumentCatalog sourceCatalog = document.getDocumentCatalog();
if (sourceCatalog != null) {
PDAcroForm sourceForm = sourceCatalog.getAcroForm(null);
if (sourceForm != null) {
rearrangedDocument
.getDocumentCatalog()
.getCOSObject()
.setItem(COSName.ACRO_FORM, sourceForm.getCOSObject());
}
}
return WebResponseUtils.pdfDocToWebResponse(
rearrangedDocument,
GeneralUtils.generateFilename(
@@ -1,13 +1,9 @@
package stirling.software.SPDF.controller.api;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -30,7 +26,6 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.FormUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@@ -59,21 +54,9 @@ public class SplitPDFController {
MultipartFile file = request.getFileInput();
TempFile outputTempFile = new TempFile(tempFileManager, ".zip");
try {
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf")) {
Files.copy(
file.getInputStream(),
sourceTempFile.getPath(),
StandardCopyOption.REPLACE_EXISTING);
int totalPages;
List<Integer> pageNumbers;
boolean hasForm;
try (PDDocument document =
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
totalPages = document.getNumberOfPages();
pageNumbers = request.getPageNumbersList(document, false);
hasForm = document.getDocumentCatalog().getAcroForm(null) != null;
}
try (PDDocument document = pdfDocumentFactory.load(file)) {
int totalPages = document.getNumberOfPages();
List<Integer> pageNumbers = request.getPageNumbersList(document, false);
if (!pageNumbers.contains(totalPages - 1)) {
pageNumbers = new ArrayList<>(pageNumbers);
pageNumbers.add(totalPages - 1);
@@ -86,16 +69,33 @@ public class SplitPDFController {
String baseFilename = GeneralUtils.removeExtension(file.getOriginalFilename());
try (ZipOutputStream zipOut =
new ZipOutputStream(Files.newOutputStream(outputTempFile.getPath()))) {
if (hasForm) {
writeSplitsViaReload(
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
} else {
writeSplitsViaSharedSource(
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
int previousPageNumber = 0;
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
int splitPoint = pageNumbers.get(splitIndex);
try (PDDocument splitDocument =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(document)) {
for (int i = previousPageNumber; i <= splitPoint; i++) {
splitDocument.addPage(document.getPage(i));
log.debug("Adding page {} to split document", i);
}
previousPageNumber = splitPoint + 1;
String fileName = baseFilename + "_" + (splitIndex + 1) + ".pdf";
zipOut.putNextEntry(new ZipEntry(fileName));
splitDocument.save(zipOut);
zipOut.closeEntry();
log.debug("Wrote split document {} to zip file", fileName);
} catch (Exception e) {
ExceptionUtils.logException("document splitting and saving", e);
throw e;
}
}
}
}
log.debug(
"Successfully created zip file with split documents: {}",
outputTempFile.getPath().toString());
String zipFilename =
GeneralUtils.generateFilename(file.getOriginalFilename(), "_split.zip");
return WebResponseUtils.zipFileToWebResponse(outputTempFile, zipFilename);
@@ -104,60 +104,4 @@ public class SplitPDFController {
throw e;
}
}
private void writeSplitsViaReload(
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
throws IOException {
int previousPageNumber = 0;
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
int splitPoint = pageNumbers.get(splitIndex);
Set<Integer> keep = new HashSet<>();
for (int i = previousPageNumber; i <= splitPoint; i++) {
keep.add(i);
}
previousPageNumber = splitPoint + 1;
try (PDDocument splitDoc = pdfDocumentFactory.load(source)) {
for (int p = splitDoc.getNumberOfPages() - 1; p >= 0; p--) {
if (!keep.contains(p)) {
splitDoc.removePage(p);
}
}
FormUtils.pruneOrphanedFormFields(splitDoc);
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
} catch (Exception e) {
ExceptionUtils.logException("document splitting and saving", e);
throw e;
}
}
}
private void writeSplitsViaSharedSource(
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
throws IOException {
try (PDDocument sourceDoc = pdfDocumentFactory.load(source)) {
int previousPageNumber = 0;
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
int splitPoint = pageNumbers.get(splitIndex);
try (PDDocument splitDoc =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDoc)) {
for (int i = previousPageNumber; i <= splitPoint; i++) {
splitDoc.addPage(sourceDoc.getPage(i));
}
previousPageNumber = splitPoint + 1;
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
} catch (Exception e) {
ExceptionUtils.logException("document splitting and saving", e);
throw e;
}
}
}
}
private void writeEntry(ZipOutputStream zipOut, String baseFilename, int index, PDDocument doc)
throws IOException {
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + index + ".pdf"));
doc.save(zipOut);
zipOut.closeEntry();
}
}
@@ -1,14 +1,8 @@
package stirling.software.SPDF.controller.api;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -22,6 +16,7 @@ import org.springframework.web.multipart.MultipartFile;
import io.swagger.v3.oas.annotations.Operation;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -31,7 +26,6 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.FormUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@@ -60,40 +54,52 @@ public class SplitPdfBySizeController {
public ResponseEntity<Resource> autoSplitPdf(
@ModelAttribute SplitPdfBySizeOrCountRequest request) throws Exception {
log.debug("Starting PDF split process with request: {}", request);
MultipartFile file = request.getFileInput();
String filename = GeneralUtils.generateFilename(file.getOriginalFilename(), "");
log.debug("Base filename for output: {}", filename);
TempFile zipTempFile = new TempFile(tempFileManager, ".zip");
try {
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf");
ZipOutputStream zipOut =
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()))) {
Files.copy(
file.getInputStream(),
sourceTempFile.getPath(),
StandardCopyOption.REPLACE_EXISTING);
log.debug("Created temporary managed zip file: {}", zipTempFile.getPath());
log.debug("Creating ZIP output stream");
try (ZipOutputStream zipOut =
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()));
PDDocument sourceDocument = pdfDocumentFactory.load(file)) {
log.debug(
"Successfully loaded PDF with {} pages", sourceDocument.getNumberOfPages());
try (PDDocument sourceDocument =
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
boolean hasForm = sourceDocument.getDocumentCatalog().getAcroForm(null) != null;
List<List<Integer>> ranges = computeRanges(request, sourceDocument);
int type = request.getSplitType();
String value = request.getSplitValue();
log.debug("Split type: {}, Split value: {}", type, value);
int fileIndex = 1;
for (List<Integer> range : ranges) {
if (range.isEmpty()) {
continue;
}
if (hasForm) {
writeRangeViaReload(
sourceTempFile.getFile(), range, zipOut, filename, fileIndex++);
} else {
writeRangeViaSharedSource(
sourceDocument, range, zipOut, filename, fileIndex++);
}
}
if (type == 0) {
log.debug("Processing split by size");
long maxBytes = GeneralUtils.convertSizeToBytes(value);
log.debug("Max bytes per document: {}", maxBytes);
handleSplitBySize(sourceDocument, maxBytes, zipOut, filename);
} else if (type == 1) {
log.debug("Processing split by page count");
int pageCount = Integer.parseInt(value);
log.debug("Pages per document: {}", pageCount);
handleSplitByPageCount(sourceDocument, pageCount, zipOut, filename);
} else if (type == 2) {
log.debug("Processing split by document count");
int documentCount = Integer.parseInt(value);
log.debug("Total number of documents: {}", documentCount);
handleSplitByDocCount(sourceDocument, documentCount, zipOut, filename);
} else {
log.error("Invalid split type: {}", type);
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument",
"Invalid argument: {0}",
"split type: " + type);
}
log.debug("PDF splitting completed successfully");
}
log.debug("Returning streaming response for zip file");
return WebResponseUtils.zipFileToWebResponse(zipTempFile, filename + ".zip");
} catch (Exception e) {
ExceptionUtils.logException("PDF splitting process", e);
@@ -102,198 +108,387 @@ public class SplitPdfBySizeController {
}
}
private List<List<Integer>> computeRanges(
SplitPdfBySizeOrCountRequest request, PDDocument sourceDocument) throws IOException {
int type = request.getSplitType();
String value = request.getSplitValue();
if (type == 0) {
return computeSizeRanges(sourceDocument, GeneralUtils.convertSizeToBytes(value));
} else if (type == 1) {
return computePageCountRanges(sourceDocument, Integer.parseInt(value));
} else if (type == 2) {
return computeDocCountRanges(sourceDocument, Integer.parseInt(value));
}
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument", "Invalid argument: {0}", "split type: " + type);
}
private void writeRangeViaReload(
File sourceFile,
List<Integer> keepIndices,
ZipOutputStream zipOut,
String baseFilename,
int fileIndex)
private void handleSplitBySize(
PDDocument sourceDocument, long maxBytes, ZipOutputStream zipOut, String baseFilename)
throws IOException {
Set<Integer> keep = new HashSet<>(keepIndices);
try (PDDocument doc = pdfDocumentFactory.load(sourceFile)) {
for (int i = doc.getNumberOfPages() - 1; i >= 0; i--) {
if (!keep.contains(i)) {
doc.removePage(i);
log.debug("Starting handleSplitBySize with maxBytes={}", maxBytes);
@Getter
class DocHolder implements AutoCloseable {
private PDDocument doc;
public DocHolder(PDDocument doc) {
this.doc = doc;
}
public void setDoc(PDDocument doc) {
if (this.doc != null) {
try {
this.doc.close();
} catch (IOException e) {
log.error("Error closing document", e);
}
}
this.doc = doc;
}
@Override
public void close() throws IOException {
if (doc != null) {
doc.close();
}
}
FormUtils.pruneOrphanedFormFields(doc);
writeEntry(zipOut, baseFilename, fileIndex, doc);
}
}
private void writeRangeViaSharedSource(
PDDocument sourceDocument,
List<Integer> keepIndices,
ZipOutputStream zipOut,
String baseFilename,
int fileIndex)
throws IOException {
try (PDDocument doc =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument)) {
for (int p : keepIndices) {
doc.addPage(sourceDocument.getPage(p));
}
writeEntry(zipOut, baseFilename, fileIndex, doc);
}
}
int fileIndex = 1;
try (DocHolder holder =
new DocHolder(
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument))) {
int totalPages = sourceDocument.getNumberOfPages();
int pageAdded = 0;
private void writeEntry(
ZipOutputStream zipOut, String baseFilename, int fileIndex, PDDocument doc)
throws IOException {
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + fileIndex + ".pdf"));
doc.save(zipOut);
zipOut.closeEntry();
}
// Smart size check frequency - check more often with larger documents
int baseCheckFrequency = 5;
/** Page-index ranges each output should contain. AcroForm overhead isn't modeled. */
private List<List<Integer>> computeSizeRanges(PDDocument sourceDocument, long maxBytes)
throws IOException {
List<List<Integer>> ranges = new ArrayList<>();
List<Integer> currentRange = new ArrayList<>();
int totalPages = sourceDocument.getNumberOfPages();
int baseCheckFrequency = 5;
PDDocument scratch = new PDDocument();
try {
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
PDPage page = sourceDocument.getPage(pageIndex);
scratch.addPage(new PDPage(page.getCOSObject()));
currentRange.add(pageIndex);
log.debug("Processing page {} of {}", pageIndex + 1, totalPages);
int pageAdded = currentRange.size();
// Add the page to current document
PDPage newPage = new PDPage(page.getCOSObject());
holder.getDoc().addPage(newPage);
pageAdded++;
// Dynamic size checking based on document size and page count
boolean shouldCheckSize =
(pageAdded % baseCheckFrequency == 0)
|| (pageIndex == totalPages - 1)
|| (pageAdded >= 20);
if (!shouldCheckSize) {
continue;
}
|| (pageAdded >= 20); // Always check after 20 pages
long actualSize;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
scratch.save(out);
actualSize = out.size();
}
if (shouldCheckSize) {
log.debug("Performing size check after {} pages", pageAdded);
long actualSize;
try (ByteArrayOutputStream checkSizeStream = new ByteArrayOutputStream()) {
holder.getDoc().save(checkSizeStream);
actualSize = checkSizeStream.size();
}
log.debug(
"Current document size: {} bytes (max: {} bytes)",
actualSize,
maxBytes);
if (actualSize > maxBytes) {
if (scratch.getNumberOfPages() > 1) {
scratch.removePage(scratch.getNumberOfPages() - 1);
currentRange.remove(currentRange.size() - 1);
pageIndex--; // retry this page in the next chunk
if (actualSize > maxBytes) {
// We exceeded the limit - remove the last page and save
if (holder.getDoc().getNumberOfPages() > 1) {
holder.getDoc().removePage(holder.getDoc().getNumberOfPages() - 1);
pageIndex--; // Process this page again in the next document
log.debug("Size limit exceeded - removed last page");
}
log.debug(
"Saving document with {} pages as part {}",
holder.getDoc().getNumberOfPages(),
fileIndex);
saveDocumentToZip(holder.getDoc(), zipOut, baseFilename, fileIndex++);
holder.setDoc(new PDDocument());
pageAdded = 0;
} else if (pageIndex < totalPages - 1) {
// We're under the limit, calculate if we might fit more pages
// Try to predict how many more similar pages might fit
if (actualSize < maxBytes * 0.75 && pageAdded > 0) {
// Rather than using a ratio, look ahead to test actual upcoming pages
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
if (pagesToLookAhead > 0) {
log.debug(
"Testing {} upcoming pages for potential addition",
pagesToLookAhead);
// Create a temp document with current pages + look-ahead pages
try (PDDocument testDoc = new PDDocument()) {
// First copy existing pages
for (int i = 0; i < holder.getDoc().getNumberOfPages(); i++) {
testDoc.addPage(
new PDPage(
holder.getDoc().getPage(i).getCOSObject()));
}
// Try adding look-ahead pages one by one
int extraPagesAdded = 0;
for (int i = 0; i < pagesToLookAhead; i++) {
int testPageIndex = pageIndex + 1 + i;
PDPage testPage = sourceDocument.getPage(testPageIndex);
testDoc.addPage(new PDPage(testPage.getCOSObject()));
// Check if we're still under size
long testSize;
try (ByteArrayOutputStream testStream =
new ByteArrayOutputStream()) {
testDoc.save(testStream);
testSize = testStream.size();
}
if (testSize <= maxBytes) {
extraPagesAdded++;
log.debug(
"Test: Can add page {} (size would be {})",
testPageIndex + 1,
testSize);
} else {
log.debug(
"Test: Cannot add page {} (size would be {})",
testPageIndex + 1,
testSize);
break;
}
}
// Add the pages we verified would fit
if (extraPagesAdded > 0) {
log.debug(
"Adding {} verified pages ahead", extraPagesAdded);
for (int i = 0; i < extraPagesAdded; i++) {
int extraPageIndex = pageIndex + 1 + i;
PDPage extraPage =
sourceDocument.getPage(extraPageIndex);
holder.getDoc()
.addPage(new PDPage(extraPage.getCOSObject()));
}
pageIndex += extraPagesAdded;
pageAdded += extraPagesAdded;
}
}
}
}
}
ranges.add(new ArrayList<>(currentRange));
currentRange.clear();
scratch.close();
scratch = new PDDocument();
} else if (pageIndex < totalPages - 1 && actualSize < maxBytes * 0.75) {
int extraPagesAdded =
lookAheadFit(scratch, sourceDocument, pageIndex, maxBytes);
for (int i = 0; i < extraPagesAdded; i++) {
int extra = pageIndex + 1 + i;
scratch.addPage(new PDPage(sourceDocument.getPage(extra).getCOSObject()));
currentRange.add(extra);
}
pageIndex += extraPagesAdded;
}
}
if (!currentRange.isEmpty()) {
ranges.add(new ArrayList<>(currentRange));
// Save final document if it has any pages
if (holder.getDoc() != null && holder.getDoc().getNumberOfPages() > 0) {
log.debug(
"Saving final document with {} pages as part {}",
holder.getDoc().getNumberOfPages(),
fileIndex);
saveDocumentToZip(holder.getDoc(), zipOut, baseFilename, fileIndex++);
holder.setDoc(null);
}
}
log.debug("Completed handleSplitBySize with {} document parts created", fileIndex - 1);
}
private void handleSplitByPageCount(
PDDocument sourceDocument, int pageCount, ZipOutputStream zipOut, String baseFilename)
throws IOException {
log.debug("Starting handleSplitByPageCount with pageCount={}", pageCount);
int currentPageCount = 0;
PDDocument currentDoc = null;
int fileIndex = 1;
try {
log.debug("Creating initial output document");
try {
currentDoc = pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument);
log.debug("Successfully created initial output document");
} catch (Exception e) {
ExceptionUtils.logException("initial output document creation", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
int pageIndex = 0;
int totalPages = sourceDocument.getNumberOfPages();
log.debug("Processing {} pages", totalPages);
try {
for (PDPage page : sourceDocument.getPages()) {
pageIndex++;
log.debug("Processing page {} of {}", pageIndex, totalPages);
try {
log.debug("Adding page {} to current document", pageIndex);
currentDoc.addPage(page);
log.debug("Successfully added page {} to current document", pageIndex);
} catch (Exception e) {
log.error("Error adding page {} to current document", pageIndex, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
currentPageCount++;
log.debug("Current page count: {}/{}", currentPageCount, pageCount);
if (currentPageCount == pageCount) {
log.debug(
"Reached target page count ({}), saving current document as part {}",
pageCount,
fileIndex);
try {
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
currentDoc = null; // Document is closed by saveDocumentToZip
log.debug("Successfully saved document part {}", fileIndex - 1);
} catch (Exception e) {
log.error("Error saving document part {}", fileIndex - 1, e);
throw e;
}
try {
log.debug("Creating new document for next part");
currentDoc = new PDDocument();
log.debug("Successfully created new document");
} catch (Exception e) {
log.error("Error creating new document for next part", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
currentPageCount = 0;
log.debug("Reset current page count to 0");
}
}
} catch (Exception e) {
log.error("Error iterating through pages", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
// Add the last document if it contains any pages
try {
if (currentDoc != null && currentDoc.getPages().getCount() != 0) {
log.debug(
"Saving final document with {} pages as part {}",
currentDoc.getPages().getCount(),
fileIndex);
try {
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
currentDoc = null; // Document is closed by saveDocumentToZip
log.debug("Successfully saved final document part {}", fileIndex - 1);
} catch (Exception e) {
log.error("Error saving final document part {}", fileIndex - 1, e);
throw e;
}
} else {
log.debug("Final document has no pages, skipping");
}
} catch (Exception e) {
log.error("Error checking or saving final document", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
} finally {
scratch.close();
if (currentDoc != null) {
try {
log.debug("Closing remaining document");
currentDoc.close();
log.debug("Successfully closed remaining document");
} catch (Exception e) {
log.error("Error closing remaining document", e);
}
}
}
return ranges;
log.debug("Completed handleSplitByPageCount with {} document parts created", fileIndex - 1);
}
/** Speculatively tries up to 5 next pages; returns how many fit under {@code maxBytes}. */
private int lookAheadFit(PDDocument scratch, PDDocument source, int pageIndex, long maxBytes)
private void handleSplitByDocCount(
PDDocument sourceDocument,
int documentCount,
ZipOutputStream zipOut,
String baseFilename)
throws IOException {
int totalPages = source.getNumberOfPages();
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
if (pagesToLookAhead == 0) {
return 0;
}
log.debug("Starting handleSplitByDocCount with documentCount={}", documentCount);
int totalPageCount = sourceDocument.getNumberOfPages();
log.debug("Total pages in source document: {}", totalPageCount);
int extraPagesAdded = 0;
try (PDDocument testDoc = new PDDocument()) {
for (int i = 0; i < scratch.getNumberOfPages(); i++) {
testDoc.addPage(new PDPage(scratch.getPage(i).getCOSObject()));
}
for (int i = 0; i < pagesToLookAhead; i++) {
testDoc.addPage(new PDPage(source.getPage(pageIndex + 1 + i).getCOSObject()));
long testSize;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
testDoc.save(out);
testSize = out.size();
}
if (testSize > maxBytes) {
break;
}
extraPagesAdded++;
}
}
return extraPagesAdded;
}
int pagesPerDocument = totalPageCount / documentCount;
int extraPages = totalPageCount % documentCount;
log.debug("Pages per document: {}, Extra pages: {}", pagesPerDocument, extraPages);
private List<List<Integer>> computePageCountRanges(PDDocument sourceDocument, int pageCount) {
if (pageCount <= 0) {
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument", "Invalid argument: {0}", "page count: " + pageCount);
}
int totalPages = sourceDocument.getNumberOfPages();
List<List<Integer>> ranges = new ArrayList<>();
List<Integer> current = new ArrayList<>(pageCount);
for (int i = 0; i < totalPages; i++) {
current.add(i);
if (current.size() == pageCount) {
ranges.add(current);
current = new ArrayList<>(pageCount);
}
}
if (!current.isEmpty()) {
ranges.add(current);
}
return ranges;
}
int currentPageIndex = 0;
int fileIndex = 1;
private List<List<Integer>> computeDocCountRanges(
PDDocument sourceDocument, int documentCount) {
if (documentCount <= 0) {
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument",
"Invalid argument: {0}",
"document count: " + documentCount);
}
int totalPages = sourceDocument.getNumberOfPages();
int pagesPerDocument = totalPages / documentCount;
int extraPages = totalPages % documentCount;
List<List<Integer>> ranges = new ArrayList<>();
int cursor = 0;
for (int i = 0; i < documentCount; i++) {
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
List<Integer> range = new ArrayList<>(pagesToAdd);
for (int j = 0; j < pagesToAdd; j++) {
range.add(cursor++);
log.debug("Creating document {} of {}", i + 1, documentCount);
PDDocument currentDoc = null;
try {
currentDoc = pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument);
log.debug("Successfully created document {} of {}", i + 1, documentCount);
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
log.debug("Adding {} pages to document {}", pagesToAdd, i + 1);
for (int j = 0; j < pagesToAdd; j++) {
try {
log.debug(
"Adding page {} (index {}) to document {}",
j + 1,
currentPageIndex,
i + 1);
currentDoc.addPage(sourceDocument.getPage(currentPageIndex));
log.debug("Successfully added page {} to document {}", j + 1, i + 1);
currentPageIndex++;
} catch (Exception e) {
log.error("Error adding page {} to document {}", j + 1, i + 1, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
}
try {
log.debug("Saving document {} with {} pages", i + 1, pagesToAdd);
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
// saveDocumentToZip closes the document
currentDoc = null;
log.debug("Successfully saved document {}", i + 1);
} catch (Exception e) {
log.error("Error saving document {}", i + 1, e);
throw e;
}
} catch (Exception e) {
log.error("Error creating document {} of {}", i + 1, documentCount, e);
throw ExceptionUtils.createFileProcessingException("split", e);
} finally {
if (currentDoc != null) {
try {
currentDoc.close();
} catch (IOException e) {
log.error("Error closing document {} of {}", i + 1, documentCount, e);
}
}
}
}
log.debug("Completed handleSplitByDocCount with {} documents created", documentCount);
}
private void saveDocumentToZip(
PDDocument document, ZipOutputStream zipOut, String baseFilename, int index)
throws IOException {
log.debug("Starting saveDocumentToZip for document part {}", index);
try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
try (PDDocument doc = document) {
log.debug("Saving document part {} to byte array", index);
doc.save(outStream);
log.debug(
"Successfully saved document part {} ({} bytes)", index, outStream.size());
} catch (Exception e) {
log.error("Error saving document part {} to byte array", index, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
try {
// Create a new zip entry
String entryName = baseFilename + "_" + index + ".pdf";
log.debug("Creating ZIP entry: {}", entryName);
ZipEntry zipEntry = new ZipEntry(entryName);
zipOut.putNextEntry(zipEntry);
byte[] bytes = outStream.toByteArray();
log.debug("Writing {} bytes to ZIP entry", bytes.length);
zipOut.write(bytes);
log.debug("Closing ZIP entry");
zipOut.closeEntry();
log.debug("Successfully added document part {} to ZIP", index);
} catch (Exception e) {
log.error("Error adding document part {} to ZIP", index, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
ranges.add(range);
}
return ranges;
}
}
@@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.QuoteMode;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.http.ContentDisposition;
@@ -26,21 +26,24 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.config.swagger.CsvConversionResponse;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.SPDF.pdf.FlexibleCSVWriter;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.ConvertApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.WebResponseUtils;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.Table;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
@ConvertApi
@Slf4j
@RequiredArgsConstructor
public class ExtractCSVController {
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final TabulaTableParser tabulaTableParser;
@AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@CsvConversionResponse
@@ -55,23 +58,24 @@ public class ExtractCSVController {
try (PDDocument document = pdfDocumentFactory.load(request)) {
List<Integer> pages = request.getPageNumbersList(document, true);
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
CSVFormat format =
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
for (int pageNum : pages) {
log.info("{}", pageNum);
List<TableFragment> fragments = tabulaTableParser.parse(document, pageNum);
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
log.info("{}", pageNum);
Page page = extractor.extract(pageNum);
List<Table> tables = sea.extract(page);
for (int i = 0; i < fragments.size(); i++) {
StringWriter sw = new StringWriter();
try (CSVPrinter printer = format.print(sw)) {
for (List<String> row : fragments.get(i).rawRows()) {
printer.printRecord(row);
}
for (int i = 0; i < tables.size(); i++) {
StringWriter sw = new StringWriter();
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
csvWriter.write(sw, Collections.singletonList(tables.get(i)));
String entryName = generateEntryName(baseName, pageNum, i + 1);
csvEntries.add(new CsvEntry(entryName, sw.toString()));
}
csvEntries.add(
new CsvEntry(
generateEntryName(baseName, pageNum, i + 1), sw.toString()));
}
}
@@ -96,9 +96,7 @@ public class RedactController {
@InitBinder
public void initBinder(WebDataBinder binder) {
binder.registerCustomEditor(
List.class,
"redactions",
new StringToArrayListPropertyEditor<>(RedactionArea.class));
List.class, "redactions", new StringToArrayListPropertyEditor());
}
@AutoJobPostMapping(value = "/redact", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@@ -682,7 +682,7 @@ public class GlobalExceptionHandler {
*
* @param ex the MaxUploadSizeExceededException
* @param request the HTTP servlet request
* @return ProblemDetail with HTTP 413 CONTENT_TOO_LARGE
* @return ProblemDetail with HTTP 413 PAYLOAD_TOO_LARGE
*/
@ExceptionHandler(MaxUploadSizeExceededException.class)
public ResponseEntity<ProblemDetail> handleMaxUploadSize(
@@ -706,7 +706,7 @@ public class GlobalExceptionHandler {
getLocalizedMessage("error.fileTooLarge.title", ErrorTitles.FILE_TOO_LARGE_DEFAULT);
ProblemDetail problemDetail =
createBaseProblemDetail(HttpStatus.CONTENT_TOO_LARGE, message, request);
createBaseProblemDetail(HttpStatus.PAYLOAD_TOO_LARGE, message, request);
problemDetail.setType(URI.create(ErrorTypes.FILE_TOO_LARGE));
problemDetail.setTitle(title);
problemDetail.setProperty("title", title); // Ensure serialization
@@ -724,7 +724,7 @@ public class GlobalExceptionHandler {
problemDetail.setProperty(
"actionRequired", "Reduce the file size to be within the upload limit.");
return ResponseEntity.status(HttpStatus.CONTENT_TOO_LARGE)
return ResponseEntity.status(HttpStatus.PAYLOAD_TOO_LARGE)
.contentType(PROBLEM_JSON)
.body(problemDetail);
}
@@ -1,32 +0,0 @@
package stirling.software.SPDF.model.api.general;
import java.util.List;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.common.model.api.general.EditTextOperation;
@Data
@EqualsAndHashCode(callSuper = true)
public class EditTextRequest extends PDFWithPageNums {
@Schema(
description =
"Ordered list of find/replace operations. Each replaces every occurrence on"
+ " the selected pages, in order; later operations see the result of"
+ " earlier ones (so 'foo'->'foos' then 'foos'->'bars' turns 'foo'"
+ " into 'bars').",
requiredMode = Schema.RequiredMode.REQUIRED)
private List<EditTextOperation> edits;
@Schema(
description =
"Whether matches must be whole words (boundaries determined by non-word"
+ " characters)",
defaultValue = "false")
private Boolean wholeWordSearch;
}
@@ -40,6 +40,16 @@ public class MergePdfsRequest extends MultiplePDFFiles {
defaultValue = "false")
private boolean generateToc = false;
@Schema(
description =
"Flag indicating whether to preserve PDF/UA accessibility tags (structure"
+ " tree) in the merged output. When false (default) the merger runs in"
+ " resource-optimised mode which drops tags but uses significantly less"
+ " heap. Set true when merging tagged PDFs intended for screen readers.",
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
defaultValue = "false")
private boolean preserveAccessibility = false;
@Schema(
description =
"JSON array of client-provided IDs for each uploaded file (same order as fileInput)",
@@ -13,7 +13,8 @@ public class RotatePDFRequest extends PDFFile {
@Schema(
description =
"The clockwise angle by which to rotate all pages in the PDF file. Must be a multiple of 90.",
"The clockwise angle by which to rotate the PDF file. Must be a multiple of"
+ " 90.",
type = "integer",
requiredMode = Schema.RequiredMode.REQUIRED,
allowableValues = {"0", "90", "180", "270"})
@@ -0,0 +1,16 @@
package stirling.software.SPDF.pdf;
import org.apache.commons.csv.CSVFormat;
import technology.tabula.writers.CSVWriter;
public class FlexibleCSVWriter extends CSVWriter {
public FlexibleCSVWriter() {
super();
}
public FlexibleCSVWriter(CSVFormat csvFormat) {
super(csvFormat);
}
}
@@ -252,17 +252,6 @@ public class PdfJsonConversionService {
convertPdfToJson(file, null, false, out);
}
/**
* Converts a PDF to the editable {@link PdfJsonDocument} model. Convenience wrapper around
* {@link #convertPdfToJson(MultipartFile, OutputStream)} for callers that need to inspect or
* mutate the document in memory before round-tripping it back to PDF.
*/
public PdfJsonDocument convertPdfToJsonDocument(MultipartFile file) throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
convertPdfToJson(file, null, false, buffer);
return objectMapper.readValue(buffer.toByteArray(), PdfJsonDocument.class);
}
public void convertPdfToJson(MultipartFile file, boolean lightweight, OutputStream out)
throws IOException {
convertPdfToJson(file, null, lightweight, out);
@@ -631,13 +620,6 @@ public class PdfJsonConversionService {
}
byte[] jsonBytes = file.getBytes();
PdfJsonDocument pdfJson = objectMapper.readValue(jsonBytes, PdfJsonDocument.class);
convertJsonToPdf(pdfJson, out);
}
public void convertJsonToPdf(PdfJsonDocument pdfJson, OutputStream out) throws IOException {
if (pdfJson == null) {
throw ExceptionUtils.createNullArgumentException("document");
}
List<PdfJsonFont> fontModels = pdfJson.getFonts();
if (fontModels == null) {
@@ -2094,29 +2076,14 @@ public class PdfJsonConversionService {
byte[] toUnicodeBytes = Base64.getDecoder().decode(toUnicodeBase64);
String toUnicodeStr = new String(toUnicodeBytes, StandardCharsets.UTF_8);
// Parse ToUnicode CMap for bfchar and bfrange. Both sides go through
// parseToUnicodeCodepoint so an 8-hex-char surrogate-pair value on either side
// (e.g. "D837DF0E") is decoded as a single supplementary codepoint instead of
// overflowing Integer.parseInt and aborting the whole mapping build.
// Parse ToUnicode CMap for bfchar and bfrange
java.util.regex.Pattern bfcharPattern =
java.util.regex.Pattern.compile("<([0-9A-Fa-f]+)>\\s*<([0-9A-Fa-f]+)>");
java.util.regex.Matcher matcher = bfcharPattern.matcher(toUnicodeStr);
while (matcher.find()) {
try {
int charCode = parseToUnicodeCodepoint(matcher.group(1));
int unicode = parseToUnicodeCodepoint(matcher.group(2));
charCodeToUnicode.put(charCode, unicode);
} catch (NumberFormatException entryEx) {
// Tolerate a single malformed entry: log and skip rather than aborting the
// entire ToUnicode CMap (which would force the whole font onto the slow
// raw-bytes fallback path).
log.debug(
"Skipping malformed ToUnicode entry <{}> <{}> in font {}: {}",
matcher.group(1),
matcher.group(2),
font.getName(),
entryEx.getMessage());
}
int charCode = Integer.parseInt(matcher.group(1), 16);
int unicode = Integer.parseInt(matcher.group(2), 16);
charCodeToUnicode.put(charCode, unicode);
}
// Build JSON mapping: CharCode → CID → GID → Unicode
@@ -2169,42 +2136,6 @@ public class PdfJsonConversionService {
}
}
/**
* Parse a hex string from a PDF ToUnicode CMap into a single Unicode codepoint. Handles three
* cases: a single BMP code unit (4 hex chars), a UTF-16 surrogate pair encoding a supplementary
* codepoint above U+FFFF (8 hex chars, e.g. {@code D837DF0E} for U+1F40E), and multi-codepoint
* mappings (longer; returns the first codepoint as a best-effort representative).
*
* <p>Without this, {@code Integer.parseInt("D837DF0E", 16)} overflows because the value is ~3.6
* billion, throwing {@link NumberFormatException} and forcing the conversion to fall back to a
* raw ToUnicode payload that the JSON&rarr;PDF rebuild then fails to use efficiently.
*/
static int parseToUnicodeCodepoint(String hex) {
if (hex == null || hex.isEmpty()) {
throw new NumberFormatException("Empty ToUnicode hex value");
}
if (hex.length() <= 4) {
return Integer.parseInt(hex, 16);
}
// Treat the hex string as UTF-16BE: pairs of hex digits form bytes, four hex digits form
// one UTF-16 code unit. The PDF ToUnicode CMap convention requires an even number of bytes
// (i.e. a multiple of four hex characters) for multi-unit values.
if (hex.length() % 4 != 0) {
throw new NumberFormatException(
"ToUnicode hex value not a multiple of 4 chars: " + hex);
}
int unitCount = hex.length() / 4;
char[] units = new char[unitCount];
for (int i = 0; i < unitCount; i++) {
units[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16);
}
// codePointAt assembles a surrogate pair into a supplementary codepoint when the
// high/low surrogates appear in sequence; for any other multi-unit sequence it returns
// the first BMP codepoint, which is the right best-effort fallback for ligature
// decompositions (one charCode -> several Unicode chars).
return new String(units).codePointAt(0);
}
private PdfJsonFontCidSystemInfo extractCidSystemInfo(COSDictionary fontDictionary) {
if (fontDictionary == null) {
return null;
@@ -4325,8 +4256,12 @@ public class PdfJsonConversionService {
return 0;
}
if (font != null) {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes())) {
int count = countCodesProtected(inputStream, font::readCode);
try (InputStream inputStream = new ByteArrayInputStream(value.getBytes())) {
int count = 0;
int code;
while ((code = font.readCode(inputStream)) != -1) {
count++;
}
if (count > 0) {
return count;
}
@@ -4338,49 +4273,6 @@ public class PdfJsonConversionService {
return Math.max(1, bytes.length);
}
/**
* Functional accessor for {@link PDFont#readCode(InputStream)} so the bounded counting loop can
* be exercised in isolation without instantiating a {@link PDFont}.
*/
@FunctionalInterface
interface CodeReader {
int readCode(InputStream stream) throws IOException;
}
/**
* Count how many codes the supplied {@code reader} can extract from {@code inputStream}, with
* two safety nets that PDFBox's raw {@link PDFont#readCode(InputStream)} loop lacks:
*
* <ol>
* <li>Stop when the stream is empty (a corrupt CMap can otherwise loop forever returning
* successfully-matched zero-bytes from an exhausted {@link ByteArrayInputStream}).
* <li>Stop when a {@code readCode} call did not consume any bytes, even if it returned a
* non-{@code -1} value.
* </ol>
*
* <p>Both conditions were observed in the wild on round-tripped fallback fonts where the
* embedded ToUnicode CMap matched 0x00 sequences, hanging the JSON&rarr;PDF rebuild.
*/
static int countCodesProtected(ByteArrayInputStream inputStream, CodeReader reader)
throws IOException {
int count = 0;
int previousAvailable = inputStream.available();
while (previousAvailable > 0) {
int code = reader.readCode(inputStream);
if (code == -1) {
break;
}
int currentAvailable = inputStream.available();
if (currentAvailable >= previousAvailable) {
// No progress made; break to avoid infinite loop on corrupt CMaps.
break;
}
count++;
previousAvailable = currentAvailable;
}
return count;
}
private MergedText mergeText(List<PdfJsonTextElement> elements) {
StringBuilder builder = new StringBuilder();
List<Integer> combinedCodes = new ArrayList<>();
File diff suppressed because it is too large Load Diff
@@ -1,664 +0,0 @@
package stirling.software.SPDF.controller.api;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.api.general.EditTextRequest;
import stirling.software.SPDF.model.json.PdfJsonDocument;
import stirling.software.SPDF.model.json.PdfJsonPage;
import stirling.software.SPDF.model.json.PdfJsonTextElement;
import stirling.software.SPDF.service.PdfJsonConversionService;
import stirling.software.common.model.api.general.EditTextOperation;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@ExtendWith(MockitoExtension.class)
class EditTextControllerTest {
@Mock private PdfJsonConversionService pdfJsonConversionService;
@Mock private TempFileManager tempFileManager;
@InjectMocks private EditTextController controller;
@BeforeEach
void setUp() throws Exception {
lenient()
.when(tempFileManager.createManagedTempFile(anyString()))
.thenAnswer(
inv -> {
File f =
Files.createTempFile("test", inv.<String>getArgument(0))
.toFile();
TempFile tf = mock(TempFile.class);
lenient().when(tf.getFile()).thenReturn(f);
lenient().when(tf.getPath()).thenReturn(f.toPath());
return tf;
});
}
private static MultipartFile pdfFile() {
return new MockMultipartFile(
"fileInput", "doc.pdf", "application/pdf", "stub-pdf-bytes".getBytes());
}
private static EditTextOperation edit(String find, String replace) {
EditTextOperation op = new EditTextOperation();
op.setFind(find);
op.setReplace(replace);
return op;
}
private static PdfJsonTextElement textElement(String text) {
PdfJsonTextElement el = new PdfJsonTextElement();
el.setText(text);
el.setCharCodes(new int[] {1, 2, 3});
return el;
}
private static PdfJsonDocument documentWithElements(List<PdfJsonTextElement> elements) {
PdfJsonDocument doc = new PdfJsonDocument();
PdfJsonPage page = new PdfJsonPage();
page.setPageNumber(1);
page.setTextElements(new ArrayList<>(elements));
doc.setPages(new ArrayList<>(List.of(page)));
return doc;
}
private static PdfJsonDocument documentWithText(String... textsByPage) {
PdfJsonDocument doc = new PdfJsonDocument();
List<PdfJsonPage> pages = new ArrayList<>();
for (int i = 0; i < textsByPage.length; i++) {
PdfJsonPage page = new PdfJsonPage();
page.setPageNumber(i + 1);
PdfJsonTextElement el = new PdfJsonTextElement();
el.setText(textsByPage[i]);
el.setCharCodes(new int[] {1, 2, 3});
page.setTextElements(new ArrayList<>(List.of(el)));
pages.add(page);
}
doc.setPages(pages);
return doc;
}
/** Build a minimal real PDF so tests can run end-to-end without mocking the conversion. */
private static byte[] buildEmptyPdf() throws Exception {
try (PDDocument doc = new PDDocument();
ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
doc.addPage(new PDPage());
doc.save(baos);
return baos.toByteArray();
}
}
@Test
void editText_nullFileInputThrows() {
EditTextRequest request = new EditTextRequest();
request.setFileInput(null);
request.setEdits(List.of(edit("foo", "bar")));
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
}
@Test
void editText_emptyEditsThrows() {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of());
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
}
@Test
void editText_nullEditsThrows() {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(null);
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
}
@Test
void editText_emptyFindStringThrows() {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("", "replacement")));
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
}
@Test
void editText_findStringWithRegexMetacharsIsTreatedLiterally() throws Exception {
// Find strings are always treated as literals (Pattern.quote'd internally) — no ReDoS
// exposure from regex metacharacters supplied by the caller.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("(unclosed", "fixed")));
PdfJsonDocument input = documentWithText("text with (unclosed paren");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
assertEquals(
"text with fixed paren",
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_literalFindReplace_mutatesMatchingSpansAndClearsCharCodes() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo", "bar")));
PdfJsonDocument input = documentWithText("foo and foo", "no match here");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
ResponseEntity<Resource> response = controller.editText(request);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getBody());
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
PdfJsonDocument mutated = captor.getValue();
assertEquals("bar and bar", mutated.getPages().get(0).getTextElements().get(0).getText());
assertNull(mutated.getPages().get(0).getTextElements().get(0).getCharCodes());
assertEquals("no match here", mutated.getPages().get(1).getTextElements().get(0).getText());
// Char codes preserved on unmodified spans.
assertNotNull(mutated.getPages().get(1).getTextElements().get(0).getCharCodes());
}
@Test
void editText_wholeWordSearch_doesNotMatchInsideWords() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("cat", "dog")));
request.setWholeWordSearch(true);
PdfJsonDocument input = documentWithText("the cat is in the catalogue");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
assertEquals(
"the dog is in the catalogue",
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_wholeWordSearch_matchesFindStartingWithNonWordChar() throws Exception {
// Regression: \b only fires on a word/non-word transition. A find that starts with a
// non-word char (e.g. "-foo") preceded by another non-word char in the source (a space)
// would never match under \b. The lookaround-based bound handles this correctly.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("-foo", "-bar")));
request.setWholeWordSearch(true);
PdfJsonDocument input = documentWithText("space then -foo here");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
assertEquals(
"space then -bar here",
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_wholeWordSearch_doesNotMatchWhenAdjacentToWordChar() throws Exception {
// The lookaround bound must still reject matches that are part of a larger word.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("-foo", "-bar")));
request.setWholeWordSearch(true);
PdfJsonDocument input = documentWithText("inline-foo should not match");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
// "inline-foo" has 'e' (word) before '-foo' so the lookbehind blocks the match. The
// trailing 'o' is followed by a space (non-word) so the trailing lookahead would pass on
// its own; the leading lookbehind is what rejects it.
assertEquals(
"inline-foo should not match",
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_pageFilter_onlyAffectsListedPages() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo", "bar")));
request.setPageNumbers("2");
PdfJsonDocument input = documentWithText("foo on page 1", "foo on page 2", "foo on page 3");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
PdfJsonDocument mutated = captor.getValue();
assertEquals("foo on page 1", mutated.getPages().get(0).getTextElements().get(0).getText());
assertEquals("bar on page 2", mutated.getPages().get(1).getTextElements().get(0).getText());
assertEquals("foo on page 3", mutated.getPages().get(2).getTextElements().get(0).getText());
}
@Test
void editText_pageRange_appliesToAllPagesInRange() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo", "bar")));
request.setPageNumbers("2-3");
PdfJsonDocument input =
documentWithText("foo page 1", "foo page 2", "foo page 3", "foo page 4");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
PdfJsonDocument mutated = captor.getValue();
assertTrue(mutated.getPages().get(0).getTextElements().get(0).getText().startsWith("foo"));
assertTrue(mutated.getPages().get(1).getTextElements().get(0).getText().startsWith("bar"));
assertTrue(mutated.getPages().get(2).getTextElements().get(0).getText().startsWith("bar"));
assertTrue(mutated.getPages().get(3).getTextElements().get(0).getText().startsWith("foo"));
}
@Test
void editText_orderedEdits_applyInSequence() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo", "bar"), edit("bar", "baz")));
PdfJsonDocument input = documentWithText("foo");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
assertEquals("baz", captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_replaceWithEmptyString_deletesMatch() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("DRAFT ", "")));
PdfJsonDocument input = documentWithText("DRAFT Confidential Memo");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
assertEquals(
"Confidential Memo",
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_noMatches_returnsPdfWithoutMutation() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("notfound", "replacement")));
PdfJsonDocument input = documentWithText("nothing to match");
int[] originalCodes = input.getPages().get(0).getTextElements().get(0).getCharCodes();
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
ResponseEntity<Resource> response = controller.editText(request);
assertEquals(HttpStatus.OK, response.getStatusCode());
// Char codes left intact when nothing was replaced.
assertEquals(
originalCodes, input.getPages().get(0).getTextElements().get(0).getCharCodes());
assertEquals(
"nothing to match", input.getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_dollarInLiteralReplacement_isQuoted() throws Exception {
// Without quoting, '$1' would be interpreted as a backreference and crash.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("price", "$100")));
PdfJsonDocument input = documentWithText("the price is set");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
assertEquals(
"the $100 is set",
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
}
@Test
void editText_emptyDocument_returnsResponseWithoutErrors() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo", "bar")));
PdfJsonDocument input = new PdfJsonDocument();
input.setPages(new ArrayList<>());
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
ResponseEntity<Resource> response = controller.editText(request);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getBody());
}
@Test
void editText_textElementWithNullText_isSkipped() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo", "bar")));
PdfJsonDocument input = new PdfJsonDocument();
PdfJsonPage page = new PdfJsonPage();
page.setPageNumber(1);
PdfJsonTextElement nullText = new PdfJsonTextElement();
nullText.setText(null);
PdfJsonTextElement realText = new PdfJsonTextElement();
realText.setText("foo here");
page.setTextElements(new ArrayList<>(List.of(nullText, realText)));
input.setPages(new ArrayList<>(List.of(page)));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
PdfJsonDocument mutated = captor.getValue();
assertNull(mutated.getPages().get(0).getTextElements().get(0).getText());
assertEquals("bar here", mutated.getPages().get(0).getTextElements().get(1).getText());
}
@Test
void editText_crossElement_matchSpansTwoElements() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
PdfJsonDocument input =
documentWithElements(List.of(textElement("Hello "), textElement("World")));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
// Whole replacement lands in the first matched element; the second is emptied. The
// JSON->PDF rebuild concatenates per-token text so the font lays out the replacement as
// one run anchored at the first element's X position.
assertEquals("Goodbye Earth", elements.get(0).getText());
assertEquals("", elements.get(1).getText());
assertNull(elements.get(0).getCharCodes());
assertNull(elements.get(1).getCharCodes());
}
@Test
void editText_crossElement_matchSpansFiveElementsLikeFragmentedTitle() throws Exception {
// Reproduces the real-world case: a multi-word title fragmented one word per text span.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(
List.of(edit("The Free Adobe Acrobat Alternative", "The PDF automation pipeline")));
PdfJsonDocument input =
documentWithElements(
List.of(
textElement("The "),
textElement("Free "),
textElement("Adobe "),
textElement("Acrobat "),
textElement("Alternative")));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
// Whole replacement is written into the first matched element; the remaining four are
// emptied. Centered titles will become left-aligned at the original first-word X position.
assertEquals("The PDF automation pipeline", elements.get(0).getText());
for (int i = 1; i < elements.size(); i++) {
assertEquals("", elements.get(i).getText(), "element " + i + " should be empty");
}
}
@Test
void editText_crossElement_preservesPrefixAndSuffix() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
PdfJsonDocument input =
documentWithElements(
List.of(textElement("Greeting: Hello "), textElement("World! And more")));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
// First element keeps its prefix and gets the entire replacement; last element keeps its
// suffix only.
assertEquals("Greeting: Goodbye Earth", elements.get(0).getText());
assertEquals("! And more", elements.get(1).getText());
}
@Test
void editText_matchInOneElementOfMany_onlyTouchesThatElement() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("World", "Earth")));
PdfJsonDocument input =
documentWithElements(
List.of(
textElement("Hello "),
textElement("World!"),
textElement(" Goodbye")));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
assertEquals("Hello ", elements.get(0).getText());
assertEquals("Earth!", elements.get(1).getText());
assertEquals(" Goodbye", elements.get(2).getText());
// Only the modified element's char codes get cleared.
assertNotNull(elements.get(0).getCharCodes());
assertNull(elements.get(1).getCharCodes());
assertNotNull(elements.get(2).getCharCodes());
}
@Test
void editText_crossElement_multipleMatchesAppliedRightToLeft() throws Exception {
// Two matches in the same page text. Right-to-left application keeps earlier indices
// valid as later matches are written.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("foo bar", "X")));
PdfJsonDocument input =
documentWithElements(
List.of(
textElement("foo "),
textElement("bar baz "),
textElement("foo "),
textElement("bar")));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
// Joined was "foo bar baz foo bar"; both "foo bar" runs replaced with "X".
StringBuilder joined = new StringBuilder();
for (PdfJsonTextElement el : elements) {
joined.append(el.getText() == null ? "" : el.getText());
}
assertEquals("X baz X", joined.toString());
}
@Test
void editText_subWordFragmentation_writesIntoFirstElement() throws Exception {
// When the matched text is split into many character-level spans (typical of Type3 glyph
// runs), the whole replacement is dumped into the first matched element and the rest are
// emptied. The font lays out the replacement as one run at the first glyph's X position.
EditTextRequest request = new EditTextRequest();
request.setFileInput(pdfFile());
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
// 11 sub-word elements covering "Hello World".
PdfJsonDocument input =
documentWithElements(
List.of(
textElement("H"),
textElement("e"),
textElement("l"),
textElement("l"),
textElement("o"),
textElement(" "),
textElement("W"),
textElement("o"),
textElement("r"),
textElement("l"),
textElement("d")));
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
controller.editText(request);
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
org.mockito.Mockito.verify(pdfJsonConversionService)
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
// Entire replacement goes into the first matched element; the other 10 are emptied.
assertEquals("Goodbye Earth", elements.get(0).getText());
for (int i = 1; i < elements.size(); i++) {
assertEquals("", elements.get(i).getText(), "element " + i + " should be empty");
}
}
@Test
void editText_outputFilenameDerivedFromInput() throws Exception {
EditTextRequest request = new EditTextRequest();
request.setFileInput(
new MockMultipartFile(
"fileInput", "report.pdf", "application/pdf", buildEmptyPdf()));
request.setEdits(List.of(edit("anything", "x")));
PdfJsonDocument input = documentWithText("nothing matches");
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
.thenReturn(input);
ResponseEntity<Resource> response = controller.editText(request);
String contentDisposition =
response.getHeaders()
.getFirst(org.springframework.http.HttpHeaders.CONTENT_DISPOSITION);
assertNotNull(contentDisposition);
assertTrue(contentDisposition.contains("report_edited.pdf"));
assertFalse(contentDisposition.contains(".pdf_edited.pdf"));
}
}
@@ -75,137 +75,57 @@ class MergeControllerTest {
void testAddTableOfContents_WithMultipleFiles_Success() throws Exception {
// Given
MultipartFile[] files = {mockFile1, mockFile2, mockFile3};
int[] pageCounts = {2, 2, 2};
// Mock the merged document setup
when(mockMergedDocument.getDocumentCatalog()).thenReturn(mockCatalog);
when(mockMergedDocument.getNumberOfPages()).thenReturn(6);
when(mockMergedDocument.getPage(0)).thenReturn(mockPage1);
when(mockMergedDocument.getPage(2)).thenReturn(mockPage2);
when(mockMergedDocument.getPage(4)).thenReturn(mockPage1);
// Mock individual document loading for page count
PDDocument doc1 = mock(PDDocument.class);
PDDocument doc2 = mock(PDDocument.class);
PDDocument doc3 = mock(PDDocument.class);
when(pdfDocumentFactory.load(mockFile1)).thenReturn(doc1);
when(pdfDocumentFactory.load(mockFile2)).thenReturn(doc2);
when(pdfDocumentFactory.load(mockFile3)).thenReturn(doc3);
when(doc1.getNumberOfPages()).thenReturn(2);
when(doc2.getNumberOfPages()).thenReturn(2);
when(doc3.getNumberOfPages()).thenReturn(2);
// When
Method addTableOfContentsMethod =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class);
addTableOfContentsMethod.setAccessible(true);
addTableOfContentsMethod.invoke(mergeController, mockMergedDocument, files);
invokeAddToc(mockMergedDocument, files, pageCounts);
// Then
ArgumentCaptor<PDDocumentOutline> outlineCaptor =
ArgumentCaptor.forClass(PDDocumentOutline.class);
verify(mockCatalog).setDocumentOutline(outlineCaptor.capture());
assertNotNull(outlineCaptor.getValue());
PDDocumentOutline capturedOutline = outlineCaptor.getValue();
assertNotNull(capturedOutline);
// Verify that documents were loaded for page count
verify(pdfDocumentFactory).load(mockFile1);
verify(pdfDocumentFactory).load(mockFile2);
verify(pdfDocumentFactory).load(mockFile3);
// Verify document closing
verify(doc1).close();
verify(doc2).close();
verify(doc3).close();
// TOC must NOT re-open source PDFs to count pages — that was the OOM hot spot.
verifyNoInteractions(pdfDocumentFactory);
}
@Test
void testAddTableOfContents_WithSingleFile_Success() throws Exception {
// Given
MultipartFile[] files = {mockFile1};
int[] pageCounts = {3};
when(mockMergedDocument.getDocumentCatalog()).thenReturn(mockCatalog);
when(mockMergedDocument.getNumberOfPages()).thenReturn(3);
when(mockMergedDocument.getPage(0)).thenReturn(mockPage1);
PDDocument doc1 = mock(PDDocument.class);
when(pdfDocumentFactory.load(mockFile1)).thenReturn(doc1);
when(doc1.getNumberOfPages()).thenReturn(3);
invokeAddToc(mockMergedDocument, files, pageCounts);
// When
Method addTableOfContentsMethod =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class);
addTableOfContentsMethod.setAccessible(true);
addTableOfContentsMethod.invoke(mergeController, mockMergedDocument, files);
// Then
verify(mockCatalog).setDocumentOutline(any(PDDocumentOutline.class));
verify(pdfDocumentFactory).load(mockFile1);
verify(doc1).close();
verifyNoInteractions(pdfDocumentFactory);
}
@Test
void testAddTableOfContents_WithEmptyArray_Success() throws Exception {
// Given
MultipartFile[] files = {};
int[] pageCounts = {};
when(mockMergedDocument.getDocumentCatalog()).thenReturn(mockCatalog);
// When
Method addTableOfContentsMethod =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class);
addTableOfContentsMethod.setAccessible(true);
addTableOfContentsMethod.invoke(mergeController, mockMergedDocument, files);
invokeAddToc(mockMergedDocument, files, pageCounts);
// Then
verify(mockMergedDocument).getDocumentCatalog();
verify(mockCatalog).setDocumentOutline(any(PDDocumentOutline.class));
verifyNoInteractions(pdfDocumentFactory);
}
@Test
void testAddTableOfContents_WithIOException_HandlesGracefully() throws Exception {
// Given
MultipartFile[] files = {mockFile1, mockFile2};
when(mockMergedDocument.getDocumentCatalog()).thenReturn(mockCatalog);
when(mockMergedDocument.getNumberOfPages()).thenReturn(4);
when(mockMergedDocument.getPage(anyInt()))
.thenReturn(mockPage1); // Use anyInt() to avoid stubbing conflicts
// First document loads successfully
PDDocument doc1 = mock(PDDocument.class);
when(pdfDocumentFactory.load(mockFile1)).thenReturn(doc1);
when(doc1.getNumberOfPages()).thenReturn(2);
// Second document throws IOException
when(pdfDocumentFactory.load(mockFile2))
.thenThrow(new IOException("Failed to load document"));
// When
Method addTableOfContentsMethod =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class);
addTableOfContentsMethod.setAccessible(true);
// Should not throw exception
assertDoesNotThrow(
() -> addTableOfContentsMethod.invoke(mergeController, mockMergedDocument, files));
// Then
verify(mockCatalog).setDocumentOutline(any(PDDocumentOutline.class));
verify(pdfDocumentFactory).load(mockFile1);
verify(pdfDocumentFactory).load(mockFile2);
verify(doc1).close();
}
@Test
void testAddTableOfContents_FilenameWithoutExtension_UsesFullName() throws Exception {
// Given
MockMultipartFile fileWithoutExtension =
new MockMultipartFile(
"file",
@@ -213,53 +133,40 @@ class MergeControllerTest {
MediaType.APPLICATION_PDF_VALUE,
"PDF content".getBytes());
MultipartFile[] files = {fileWithoutExtension};
int[] pageCounts = {1};
when(mockMergedDocument.getDocumentCatalog()).thenReturn(mockCatalog);
when(mockMergedDocument.getNumberOfPages()).thenReturn(1);
when(mockMergedDocument.getPage(0)).thenReturn(mockPage1);
PDDocument doc = mock(PDDocument.class);
when(pdfDocumentFactory.load(fileWithoutExtension)).thenReturn(doc);
when(doc.getNumberOfPages()).thenReturn(1);
invokeAddToc(mockMergedDocument, files, pageCounts);
// When
Method addTableOfContentsMethod =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class);
addTableOfContentsMethod.setAccessible(true);
addTableOfContentsMethod.invoke(mergeController, mockMergedDocument, files);
// Then
verify(mockCatalog).setDocumentOutline(any(PDDocumentOutline.class));
verify(doc).close();
verifyNoInteractions(pdfDocumentFactory);
}
@Test
void testAddTableOfContents_PageIndexExceedsDocumentPages_HandlesGracefully() throws Exception {
// Given
MultipartFile[] files = {mockFile1};
int[] pageCounts = {3};
when(mockMergedDocument.getDocumentCatalog()).thenReturn(mockCatalog);
when(mockMergedDocument.getNumberOfPages()).thenReturn(0); // No pages in merged document
PDDocument doc1 = mock(PDDocument.class);
when(pdfDocumentFactory.load(mockFile1)).thenReturn(doc1);
when(doc1.getNumberOfPages()).thenReturn(3);
assertDoesNotThrow(() -> invokeAddToc(mockMergedDocument, files, pageCounts));
// When
Method addTableOfContentsMethod =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class);
addTableOfContentsMethod.setAccessible(true);
// Should not throw exception
assertDoesNotThrow(
() -> addTableOfContentsMethod.invoke(mergeController, mockMergedDocument, files));
// Then
verify(mockCatalog).setDocumentOutline(any(PDDocumentOutline.class));
verify(mockMergedDocument, never()).getPage(anyInt());
verify(doc1).close();
verifyNoInteractions(pdfDocumentFactory);
}
private void invokeAddToc(PDDocument merged, MultipartFile[] files, int[] pageCounts)
throws Exception {
Method m =
MergeController.class.getDeclaredMethod(
"addTableOfContents", PDDocument.class, MultipartFile[].class, int[].class);
m.setAccessible(true);
m.invoke(mergeController, merged, files, pageCounts);
}
@Test
@@ -3,10 +3,8 @@ package stirling.software.SPDF.controller.api;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -26,6 +24,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.api.SplitPagesRequest;
import stirling.software.common.service.CustomPDFDocumentFactory;
@@ -61,10 +60,8 @@ class SplitPDFControllerTest {
}
private void setupFactory() throws IOException {
when(pdfDocumentFactory.load(any(File.class), eq(true)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(File.class)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(MultipartFile.class)))
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
.thenAnswer(inv -> new PDDocument());
}
@@ -3,10 +3,8 @@ package stirling.software.SPDF.controller.api;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -27,6 +25,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.api.general.SplitPdfBySizeOrCountRequest;
import stirling.software.common.service.CustomPDFDocumentFactory;
@@ -71,8 +70,9 @@ class SplitPdfBySizeControllerTest {
request.setSplitType(1); // Page count
request.setSplitValue("2");
when(pdfDocumentFactory.load(any(File.class), eq(true)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(MultipartFile.class)))
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
.thenAnswer(inv -> new PDDocument());
@@ -104,8 +104,14 @@ class SplitPdfBySizeControllerTest {
request.setSplitType(2); // Document count
request.setSplitValue("3"); // Split into 3 docs (2 pages each)
when(pdfDocumentFactory.load(any(File.class), eq(true)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(org.springframework.web.multipart.MultipartFile.class)))
.thenAnswer(
inv ->
Loader.loadPDF(
((org.springframework.web.multipart.MultipartFile)
inv.getArgument(0))
.getBytes()));
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
.thenAnswer(inv -> new PDDocument());
@@ -20,7 +20,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.GeneralUtils;
@@ -28,7 +27,6 @@ import stirling.software.common.util.GeneralUtils;
class ExtractCSVControllerTest {
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
@Mock private TabulaTableParser tabulaTableParser;
@InjectMocks private ExtractCSVController controller;
@@ -177,14 +177,14 @@ class GlobalExceptionHandlerTest {
void handleMaxUploadSize_returns_413() {
MaxUploadSizeExceededException ex = new MaxUploadSizeExceededException(10485760);
ResponseEntity<ProblemDetail> resp = handler.handleMaxUploadSize(ex, request);
assertEquals(HttpStatus.CONTENT_TOO_LARGE, resp.getStatusCode());
assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, resp.getStatusCode());
}
@Test
void handleMaxUploadSize_unknown_limit() {
MaxUploadSizeExceededException ex = new MaxUploadSizeExceededException(-1);
ResponseEntity<ProblemDetail> resp = handler.handleMaxUploadSize(ex, request);
assertEquals(HttpStatus.CONTENT_TOO_LARGE, resp.getStatusCode());
assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, resp.getStatusCode());
assertNull(resp.getBody().getProperties().get("maxSizeBytes"));
}
@@ -0,0 +1,25 @@
package stirling.software.SPDF.pdf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.commons.csv.CSVFormat;
import org.junit.jupiter.api.Test;
class FlexibleCSVWriterTest {
@Test
void testDefaultConstructor() {
FlexibleCSVWriter writer = new FlexibleCSVWriter();
assertNotNull(writer, "The FlexibleCSVWriter instance should not be null");
}
@Test
void testConstructorWithCSVFormat() {
CSVFormat csvFormat = CSVFormat.DEFAULT;
FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat);
assertNotNull(
writer,
"The FlexibleCSVWriter instance should not be null when initialized with"
+ " CSVFormat");
}
}
@@ -1,161 +0,0 @@
package stirling.software.SPDF.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import org.junit.jupiter.api.Test;
/**
* Unit tests for {@link PdfJsonConversionService#parseToUnicodeCodepoint(String)}.
*
* <p>The function exists because PDF ToUnicode CMap entries can encode supplementary-plane
* codepoints (above U+FFFF) as UTF-16 surrogate pairs, e.g. {@code <D837DF0E>} for U+1F40E. The
* naive implementation that called {@link Integer#parseInt(String, int)} on the whole hex string
* threw {@link NumberFormatException} for any 8-char value because they overflow {@code int}. That
* triggered a fallback that left fonts with no proper CID mapping and made downstream JSON&rarr;PDF
* rebuilds slow/hung when rendering text with those fonts.
*/
class PdfJsonConversionServiceUnicodeParsingTest {
@Test
void parsesSingleBmpCodeUnit() {
// Latin capital A.
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("0041"));
}
@Test
void parsesShortBmpHexValue() {
// Some ToUnicode entries omit leading zeros for low codepoints.
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("41"));
}
@Test
void parsesSupplementaryCodepointFromSurrogatePair() {
// U+1F40E HORSE encoded as UTF-16 surrogate pair D83D DC0E. The bug we are fixing was
// that Integer.parseInt("D83DDC0E", 16) overflows because D83DDC0E > Integer.MAX_VALUE.
assertEquals(0x1F40E, PdfJsonConversionService.parseToUnicodeCodepoint("D83DDC0E"));
}
@Test
void parsesSurrogatePairFromUserReportedHang() {
// The exact hex value from the user's hang reproducer log:
// "Failed to build Unicode mapping for font NotoSans-Regular: For input string:
// \"D837DF0E\" under radix 16"
// D837 DF0E decodes to U+1DF0E (CJK supplementary). Important assertion: it returns a
// valid codepoint instead of overflowing Integer.parseInt and throwing.
int expected = new String(new char[] {(char) 0xD837, (char) 0xDF0E}).codePointAt(0);
assertEquals(0x1DF0E, expected); // sanity check on the test setup itself
assertEquals(expected, PdfJsonConversionService.parseToUnicodeCodepoint("D837DF0E"));
}
@Test
void parsesLigatureDecompositionAsFirstCodepoint() {
// A ToUnicode entry can map one charCode to multiple Unicode chars (a ligature). PDF
// spec allows e.g. <0041 0042> for "AB". Our best-effort behavior is to return the
// first codepoint so the mapping is at least roughly meaningful for search/copy.
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("00410042"));
}
@Test
void rejectsEmptyHex() {
assertThrows(
NumberFormatException.class,
() -> PdfJsonConversionService.parseToUnicodeCodepoint(""));
}
@Test
void rejectsNullHex() {
assertThrows(
NumberFormatException.class,
() -> PdfJsonConversionService.parseToUnicodeCodepoint(null));
}
@Test
void rejectsOddByteCountAboveBmp() {
// 6 hex chars is 3 bytes — not a valid UTF-16BE sequence.
assertThrows(
NumberFormatException.class,
() -> PdfJsonConversionService.parseToUnicodeCodepoint("D83DDC"));
}
@Test
void countCodesProtectedTerminatesWhenReaderMakesNoProgress() {
// Reproduces the user's hang: PDFBox's CMap.readCode can return a successful code (0)
// from a stream where no bytes were consumed (corrupt codespace matching 0x00 bytes from
// the buffer's uninitialized region after EOF). Without the no-progress guard, the
// counting loop in countGlyphs ran forever.
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3, 4});
PdfJsonConversionService.CodeReader reader = in -> 0; // never reads, always "succeeds"
int count =
assertTimeoutPreemptively(
Duration.ofSeconds(2),
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
// First iteration sees no progress and breaks immediately.
assertEquals(0, count);
}
@Test
void countCodesProtectedTerminatesOnEmptyStream() {
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]);
PdfJsonConversionService.CodeReader reader =
in -> {
throw new AssertionError("reader must not be called when stream is empty");
};
int count =
assertTimeoutPreemptively(
Duration.ofSeconds(2),
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
assertEquals(0, count);
}
@Test
void countCodesProtectedHonorsExplicitMinusOneReturn() throws IOException {
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3});
PdfJsonConversionService.CodeReader reader =
in -> {
int b = in.read();
return b == -1 ? -1 : b;
};
int count = PdfJsonConversionService.countCodesProtected(stream, reader);
assertEquals(3, count);
}
@Test
void countCodesProtectedTerminatesIfReaderReadsThenStops() throws IOException {
// A reader that consumes one byte then hits a corrupt-CMap pattern returning 0 without
// consuming further must still terminate after counting the consumed bytes.
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3, 4});
PdfJsonConversionService.CodeReader reader =
new PdfJsonConversionService.CodeReader() {
boolean firstCall = true;
@Override
public int readCode(InputStream in) throws IOException {
if (firstCall) {
firstCall = false;
return in.read();
}
return 0; // simulates corrupt CMap thereafter
}
};
int count =
assertTimeoutPreemptively(
Duration.ofSeconds(2),
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
assertEquals(1, count);
}
}
+6
View File
@@ -59,6 +59,12 @@ dependencies {
// https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
// Tabula table extraction — used by MathAuditorOrchestrator
implementation ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
implementation 'com.google.code.gson:gson:2.13.2'
api 'io.micrometer:micrometer-registry-prometheus'
@@ -30,7 +30,6 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.service.TaskManager;
import stirling.software.proprietary.model.api.ai.AiWorkflowProgressEvent;
import stirling.software.proprietary.model.api.ai.AiWorkflowRequest;
import stirling.software.proprietary.model.api.ai.AiWorkflowResponse;
import stirling.software.proprietary.model.api.ai.AiWorkflowResultFile;
@@ -141,32 +140,13 @@ public class AiEngineController {
}
private void runOrchestrationStream(AiWorkflowRequest request, SseEmitter emitter) {
AiWorkflowService.ProgressListener listener =
new AiWorkflowService.ProgressListener() {
@Override
public void onProgress(AiWorkflowProgressEvent event) {
sendEvent(emitter, "progress", event);
}
@Override
public void onHeartbeat() {
// Forward upstream heartbeats so the SSE pipe stays visibly alive between
// real progress events; if the frontend has gone away, sendEvent throws,
// which propagates up through the stream consumer and closes our upstream
// engine connection so the engine can cancel its in-flight workflow.
sendEvent(emitter, "heartbeat", Map.of());
}
};
try {
AiWorkflowResponse result = aiWorkflowService.orchestrate(request, listener);
AiWorkflowResponse result =
aiWorkflowService.orchestrate(
request, progress -> sendEvent(emitter, "progress", progress));
registerFileResultAsJob(result);
sendEvent(emitter, "result", result);
emitter.complete();
} catch (ClientDisconnectedException e) {
// The frontend gave up mid-stream. The exception unwinding through orchestrate()
// already closed the upstream engine connection (engine sees disconnect and cancels).
// The emitter is already toast; nothing useful left to send.
log.debug("Client disconnected mid-stream; aborting workflow", e);
} catch (Exception e) {
log.error("AI orchestration stream failed", e);
// Emit an error frame for the frontend and then complete normally. Using
@@ -212,21 +192,7 @@ public class AiEngineController {
try {
emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON));
} catch (IOException e) {
// Surface the disconnect so the streaming pipeline unwinds: callers higher up close
// the upstream engine connection, which lets the engine cancel its in-flight workflow.
// Without this, the engine would keep producing (and billing for) tokens whose results
// nobody is reading.
throw new ClientDisconnectedException("Client disconnected from SSE stream", e);
}
}
/**
* Thrown by {@link #sendEvent} when the SSE emitter's underlying connection is gone. Treated as
* a signal to abort the workflow, not as an error to report.
*/
private static final class ClientDisconnectedException extends RuntimeException {
ClientDisconnectedException(String message, Throwable cause) {
super(message, cause);
log.debug("Failed to send SSE event (client may have disconnected)", e);
}
}
@@ -1,24 +0,0 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Body for {@code POST /api/v1/documents} on the AI engine. Sent by Java when the engine reports
* {@code need_ingest} and the requested document's extracted content must be stored before the
* workflow can continue.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AiDocumentIngestRequest {
private String documentId;
private String source;
private List<AiPageText> pageText;
}
@@ -1,59 +0,0 @@
package stirling.software.proprietary.model.api.ai;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* Typed engine-emitted progress detail, mirroring the Python {@code ProgressEvent} discriminated
* union (see {@code engine/src/stirling/contracts/progress.py}). Carried inside {@link
* AiWorkflowProgressEvent#getEngineDetail()} for {@link AiWorkflowPhase#ENGINE_PROGRESS} events.
*
* <p>Sealed so adding a new engine-side phase forces a matching subtype on the Java side instead of
* silently passing through as an opaque map. The {@code phase} string is the discriminator and
* stays on the wire so the frontend (which doesn't know about Java's class hierarchy) can switch on
* it.
*/
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.EXISTING_PROPERTY,
property = "phase",
visible = true)
@JsonSubTypes({
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocReadStarted.class,
name = "whole_doc_read_started"),
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocSliceDone.class,
name = "whole_doc_slice_done"),
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocCompressionRound.class,
name = "whole_doc_compression_round"),
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocReadDone.class,
name = "whole_doc_read_done"),
})
@JsonIgnoreProperties(ignoreUnknown = true)
public sealed interface AiEngineProgressDetail {
String phase();
record WholeDocReadStarted(String phase, String question, int pages, int slices)
implements AiEngineProgressDetail {}
record WholeDocSliceDone(
String phase,
int completed,
int total,
String pages,
int durationMs,
int excerpts,
int facts)
implements AiEngineProgressDetail {}
record WholeDocCompressionRound(String phase, int roundNumber, int notesIn, int groups)
implements AiEngineProgressDetail {}
record WholeDocReadDone(String phase, int completed, int slices, double durationSeconds)
implements AiEngineProgressDetail {}
}
@@ -0,0 +1,24 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Body for {@code POST /api/v1/rag/documents} on the AI engine. Sent by Java when the engine
* reports {@code need_ingest} and the requested document's extracted content must be stored before
* the workflow can continue.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AiRagIngestRequest {
private String documentId;
private String source;
private List<AiRagPageText> pageText;
}
@@ -4,11 +4,11 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/** A single page of extracted text for document ingest requests. */
/** A single page of extracted text for RAG ingest requests. */
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AiPageText {
public class AiRagPageText {
private int pageNumber;
@@ -20,8 +20,7 @@ public enum AiWorkflowOutcome {
TOOL_CALL("tool_call"),
COMPLETED("completed"),
UNSUPPORTED_CAPABILITY("unsupported_capability"),
CANNOT_CONTINUE("cannot_continue"),
GENERATE_FILE("generate_file");
CANNOT_CONTINUE("cannot_continue");
private final String value;

Some files were not shown because too many files have changed in this diff Show More