Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f651bc1014 | ||
|
|
5b9ef852ab | ||
|
|
672e81d286 | ||
|
|
8abe734f0b | ||
|
|
8a59c10f42 | ||
|
|
d62f2ad3ed | ||
|
|
f60a075443 | ||
|
|
c059e13423 | ||
|
|
575684ee4b | ||
|
|
294b616a63 | ||
|
|
f61121953e | ||
|
|
8645280ee5 | ||
|
|
bd50e2e59d | ||
|
|
77abaf5de5 | ||
|
|
503a81be50 | ||
|
|
67fc5c70b9 | ||
|
|
1e6514a471 | ||
|
|
c4874deb5d | ||
|
|
4ee47fdddd |
@@ -22,7 +22,7 @@ runs:
|
||||
steps:
|
||||
- name: Generate a GitHub App Token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
||||
with:
|
||||
app-id: ${{ inputs.app-id }}
|
||||
private-key: ${{ inputs.private-key }}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.10.1
|
||||
pkgver=2.11.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.10.1
|
||||
pkgver=2.11.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
pr_ref: ${{ steps.resolve.outputs.ref }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -359,7 +359,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -474,7 +474,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Publish stirling-pdf-desktop to AUR
|
||||
if: ${{ github.event_name == 'release' || inputs.dry_run == false }}
|
||||
uses: KSXGitHub/github-actions-deploy-aur@v4.1.2
|
||||
uses: KSXGitHub/github-actions-deploy-aur@v4.1.3
|
||||
with:
|
||||
pkgname: stirling-pdf-desktop
|
||||
pkgbuild: .github/aur/stirling-pdf-desktop/PKGBUILD
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
spring-security: [true, false]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -63,7 +63,15 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task frontend:test:e2e:install -- chromium
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (needed for playwright's vite preview webServer)
|
||||
# Enterprise tests target :8080 (Spring Boot's bundled frontend), but
|
||||
# playwright's webServer config still launches `vite preview --port 5173`
|
||||
# before any test run, which needs dist/ to exist. VITE_BUILD_FOR_PREVIEW
|
||||
# forces absolute asset paths so vite preview can serve deep SPA routes.
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
|
||||
- name: Resolve kubernetes.docker.internal to localhost
|
||||
# The compose stacks set KC_HOSTNAME=kubernetes.docker.internal so
|
||||
@@ -140,7 +148,7 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise OAuth Playwright tests
|
||||
id: oauth-tests
|
||||
run: task frontend:test:e2e -- --project=enterprise --grep "OAuth"
|
||||
run: task e2e:enterprise -- --grep "OAuth"
|
||||
- name: Stop backend + tear down OAuth Keycloak
|
||||
if: always()
|
||||
run: |
|
||||
@@ -213,7 +221,7 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise SAML Playwright tests
|
||||
id: saml-tests
|
||||
run: task frontend:test:e2e -- --project=enterprise --grep "SAML"
|
||||
run: task e2e:enterprise -- --grep "SAML"
|
||||
- name: Stop backend + tear down SAML Keycloak
|
||||
if: always()
|
||||
run: |
|
||||
@@ -243,7 +251,7 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise feature Playwright tests
|
||||
id: feature-tests
|
||||
run: task frontend:test:e2e -- --project=enterprise --grep "Enterprise license"
|
||||
run: task e2e:enterprise -- --grep "Enterprise license"
|
||||
- name: Print backend log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
pull-requests: write # Allow writing to pull requests
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -34,58 +34,26 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task frontend:test:e2e:install -- chromium
|
||||
- name: Start Spring Boot backend (background)
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
env:
|
||||
# Suppress the analytics opt-in modal that fires on first admin login when
|
||||
# enableAnalytics is null (see Onboarding.tsx). The modal renders a Mantine
|
||||
# overlay that intercepts pointer events on every tool page until dismissed,
|
||||
# which causes every "click run button" assertion in the live suite to fail.
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
# NOTE: SECURITY_INITIALLOGIN_USERNAME/PASSWORD are intentionally NOT set.
|
||||
# The live-setup project's bootstrap spec performs the real first-login
|
||||
# flow against the backend's default admin/stirling user, exercising the
|
||||
# forced-password-change UI and leaving the DB at admin/adminadmin for
|
||||
# the rest of the live suite. This is both real coverage of the first-
|
||||
# login flow and a stronger seed than env-var-driven user creation.
|
||||
run: |
|
||||
nohup ./gradlew :stirling-pdf:bootRun > /tmp/backend.log 2>&1 &
|
||||
echo $! > /tmp/backend.pid
|
||||
- name: Wait for backend to become ready
|
||||
run: |
|
||||
start=$SECONDS
|
||||
# 300 iterations × 2s = 10 minute ceiling
|
||||
for i in $(seq 1 300); do
|
||||
if curl -fsS http://localhost:8080/api/v1/info/status >/dev/null 2>&1; then
|
||||
echo "Backend up after $((SECONDS - start))s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "Backend did not become ready in $((SECONDS - start))s"
|
||||
tail -200 /tmp/backend.log || true
|
||||
exit 1
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run live E2E tests (chromium)
|
||||
id: live-tests
|
||||
run: task frontend:test:e2e -- --project=live
|
||||
run: task e2e:live
|
||||
- name: Print backend log on failure
|
||||
if: failure() && steps.live-tests.conclusion == 'failure'
|
||||
run: |
|
||||
echo "::group::Spring Boot backend log (last 500 lines)"
|
||||
tail -500 /tmp/backend.log || echo "no backend log found"
|
||||
tail -500 .test-state/playwright/backend.log || echo "no backend log found"
|
||||
echo "::endgroup::"
|
||||
- name: Stop backend
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/backend.pid ]; then
|
||||
kill "$(cat /tmp/backend.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Upload backend log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: backend-log-live-${{ github.run_id }}
|
||||
path: /tmp/backend.log
|
||||
path: .test-state/playwright/backend.log
|
||||
retention-days: 7
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -28,9 +28,13 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task frontend:test:e2e:install -- chromium
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run stubbed E2E tests (chromium)
|
||||
run: task frontend:test:e2e -- --project=stubbed --workers=3
|
||||
run: task e2e:stubbed -- --workers=3
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -316,7 +316,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -85,6 +85,26 @@ jobs:
|
||||
echo "to see what still needs fixing manually."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
- name: Remove frontend check comment on success
|
||||
if: steps.frontend-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- frontend-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
- name: Upload frontend build artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
file_suffix: "-server"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
allowed-endpoints: >
|
||||
@@ -606,7 +606,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -35,10 +35,10 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install all Playwright browsers
|
||||
run: task frontend:test:e2e:install
|
||||
run: task e2e:install
|
||||
|
||||
- name: Run E2E tests (all browsers)
|
||||
run: task frontend:test:e2e
|
||||
run: task e2e:cross-browser
|
||||
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
PIP_DISABLE_PIP_VERSION_CHECK: "1"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -2,6 +2,22 @@ name: Push Docker Image with VersionNumber
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_main_app:
|
||||
description: "Build & push the main Stirling-PDF image (latest, fat, ultra-lite)."
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
build_unoserver:
|
||||
description: "Build & push the standalone stirling-unoserver image."
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
force_unoserver_rebuild:
|
||||
description: "Rebuild stirling-unoserver even if its source hash is unchanged."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -31,9 +47,13 @@ jobs:
|
||||
permissions:
|
||||
packages: write
|
||||
id-token: write
|
||||
# On push events these stay 'true'; on workflow_dispatch they follow the inputs.
|
||||
env:
|
||||
RUN_MAIN_APP: ${{ github.event_name != 'workflow_dispatch' || inputs.build_main_app }}
|
||||
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -46,7 +66,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -108,6 +128,7 @@ jobs:
|
||||
|
||||
- name: Generate tags for latest
|
||||
id: meta
|
||||
if: env.RUN_MAIN_APP == 'true'
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
with:
|
||||
images: |
|
||||
@@ -122,6 +143,8 @@ jobs:
|
||||
|
||||
- name: Build and push Unified Dockerfile (latest variant)
|
||||
id: build-push-latest
|
||||
# Empty-tag guard: build-push-action errors when asked to push with no tags.
|
||||
if: env.RUN_MAIN_APP == 'true' && steps.meta.outputs.tags != ''
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -140,7 +163,7 @@ jobs:
|
||||
sbom: true
|
||||
|
||||
- name: Sign regular images
|
||||
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
|
||||
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-latest.outputs.digest != ''
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-latest.outputs.digest }}
|
||||
TAGS: ${{ steps.meta.outputs.tags }}
|
||||
@@ -156,7 +179,7 @@ jobs:
|
||||
- name: Generate tags for latest-fat
|
||||
id: meta-fat
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
@@ -170,7 +193,7 @@ jobs:
|
||||
- name: Build and push Unified Dockerfile (fat variant)
|
||||
id: build-push-fat
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-fat.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -186,7 +209,7 @@ jobs:
|
||||
sbom: true
|
||||
|
||||
- name: Sign fat images
|
||||
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
|
||||
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-fat.outputs.digest != ''
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-fat.outputs.digest }}
|
||||
TAGS: ${{ steps.meta-fat.outputs.tags }}
|
||||
@@ -200,7 +223,7 @@ jobs:
|
||||
- name: Generate tags for ultra-lite
|
||||
id: meta-lite
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
@@ -214,7 +237,7 @@ jobs:
|
||||
- name: Build and push Unified Dockerfile (ultra-lite variant)
|
||||
id: build-push-lite
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-lite.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -230,7 +253,7 @@ jobs:
|
||||
sbom: true
|
||||
|
||||
- name: Sign ultra-lite images
|
||||
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
|
||||
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-lite.outputs.digest != ''
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-lite.outputs.digest }}
|
||||
TAGS: ${{ steps.meta-lite.outputs.tags }}
|
||||
@@ -240,3 +263,136 @@ jobs:
|
||||
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
|
||||
# Standalone unoserver image — versioned independently via
|
||||
# docker/unoserver/VERSION. master/V2-master: publish <version>+latest
|
||||
# only when the version is new. main/testMain: republish :alpha only
|
||||
# when the source hash differs from the published image's annotation.
|
||||
- name: Read unoserver image version
|
||||
id: unoserverVersion
|
||||
if: env.RUN_UNOSERVER == 'true'
|
||||
run: |
|
||||
version=$(tr -d '[:space:]' < docker/unoserver/VERSION)
|
||||
if [ -z "$version" ]; then
|
||||
echo "docker/unoserver/VERSION is empty"; exit 1
|
||||
fi
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "Unoserver image version (from file): ${version}"
|
||||
|
||||
- name: Compute unoserver image source hash
|
||||
id: unoserverHash
|
||||
if: env.RUN_UNOSERVER == 'true'
|
||||
run: |
|
||||
set -eu
|
||||
hash=$(cat \
|
||||
docker/unoserver/Dockerfile \
|
||||
docker/unoserver/entrypoint.sh \
|
||||
docker/unoserver/healthcheck.sh \
|
||||
docker/unoserver/VERSION \
|
||||
| sha256sum | cut -d' ' -f1)
|
||||
echo "hash=${hash}" >> "$GITHUB_OUTPUT"
|
||||
echo "Unoserver source hash: ${hash}"
|
||||
|
||||
- name: Decide whether to publish unoserver image
|
||||
id: unoserverDecision
|
||||
if: env.RUN_UNOSERVER == 'true'
|
||||
env:
|
||||
UNOSERVER_VERSION: ${{ steps.unoserverVersion.outputs.version }}
|
||||
UNOSERVER_HASH: ${{ steps.unoserverHash.outputs.hash }}
|
||||
UNOSERVER_IMAGE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-unoserver
|
||||
UNOSERVER_HASH_ANNOTATION: org.stirlingpdf.unoserver-source-hash
|
||||
FORCE_REBUILD: ${{ inputs.force_unoserver_rebuild }}
|
||||
GH_REF: ${{ github.ref }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
set -eu
|
||||
mode="skip"
|
||||
tags=""
|
||||
|
||||
read_published_hash() {
|
||||
local ref="$1"
|
||||
docker buildx imagetools inspect "$ref" --raw 2>/dev/null \
|
||||
| jq -r --arg key "$UNOSERVER_HASH_ANNOTATION" \
|
||||
'.annotations[$key] // empty' \
|
||||
2>/dev/null || true
|
||||
}
|
||||
|
||||
# Manual dispatch from any branch routes to the :alpha publish path.
|
||||
EFFECTIVE_REF="$GH_REF"
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
EFFECTIVE_REF="refs/heads/testMain"
|
||||
fi
|
||||
|
||||
case "$EFFECTIVE_REF" in
|
||||
refs/heads/master|refs/heads/V2-master)
|
||||
if [ "${FORCE_REBUILD}" = "true" ]; then
|
||||
echo "force_unoserver_rebuild=true — building stable regardless"
|
||||
mode="stable"
|
||||
tags="${UNOSERVER_IMAGE}:${UNOSERVER_VERSION},${UNOSERVER_IMAGE}:latest"
|
||||
elif docker manifest inspect "${UNOSERVER_IMAGE}:${UNOSERVER_VERSION}" >/dev/null 2>&1; then
|
||||
echo "stirling-unoserver:${UNOSERVER_VERSION} already on GHCR — skipping"
|
||||
else
|
||||
echo "stirling-unoserver:${UNOSERVER_VERSION} is new — will publish"
|
||||
mode="stable"
|
||||
tags="${UNOSERVER_IMAGE}:${UNOSERVER_VERSION},${UNOSERVER_IMAGE}:latest"
|
||||
fi
|
||||
;;
|
||||
refs/heads/main|refs/heads/testMain)
|
||||
published_hash=$(read_published_hash "${UNOSERVER_IMAGE}:alpha")
|
||||
if [ "${FORCE_REBUILD}" = "true" ]; then
|
||||
echo "force_unoserver_rebuild=true — rebuilding :alpha regardless"
|
||||
mode="alpha"
|
||||
tags="${UNOSERVER_IMAGE}:alpha"
|
||||
elif [ -n "$published_hash" ] && [ "$published_hash" = "$UNOSERVER_HASH" ]; then
|
||||
echo "Published :alpha source hash matches (${published_hash}) — skipping"
|
||||
else
|
||||
if [ -z "$published_hash" ]; then
|
||||
echo ":alpha has no source-hash annotation (first publish or pre-tracking image) — will publish"
|
||||
else
|
||||
echo "Source hash changed (was ${published_hash}, now ${UNOSERVER_HASH}) — will publish"
|
||||
fi
|
||||
mode="alpha"
|
||||
tags="${UNOSERVER_IMAGE}:alpha"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Branch ${GH_REF} does not publish unoserver image"
|
||||
;;
|
||||
esac
|
||||
echo "mode=${mode}" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and push unoserver image
|
||||
id: build-push-unoserver
|
||||
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode != 'skip'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./docker/unoserver/Dockerfile
|
||||
push: true
|
||||
cache-from: type=gha,scope=stirling-unoserver
|
||||
cache-to: type=gha,mode=max,scope=stirling-unoserver
|
||||
tags: ${{ steps.unoserverDecision.outputs.tags }}
|
||||
# Manifest annotation read by the decision step above to detect drift.
|
||||
annotations: |
|
||||
index:org.stirlingpdf.unoserver-source-hash=${{ steps.unoserverHash.outputs.hash }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Sign unoserver image
|
||||
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode == 'stable'
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-unoserver.outputs.digest }}
|
||||
TAGS: ${{ steps.unoserverDecision.outputs.tags }}
|
||||
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
||||
run: |
|
||||
if [ -n "$COSIGN_PRIVATE_KEY" ]; then
|
||||
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
else
|
||||
echo "Warning: COSIGN_PRIVATE_KEY not set, skipping unoserver image signing"
|
||||
fi
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,6 +75,6 @@ jobs:
|
||||
|
||||
# Upload the results to GitHub's code scanning dashboard.
|
||||
- name: "Upload to code-scanning"
|
||||
uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v3.29.5
|
||||
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v3.29.5
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -557,7 +557,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -648,7 +648,7 @@ jobs:
|
||||
if: always()
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
cache-scope: stirling-pdf-fat
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
@@ -160,3 +160,58 @@ jobs:
|
||||
build/reports/problems/
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-build-unoserver-image:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./docker/unoserver/Dockerfile
|
||||
push: false
|
||||
load: true
|
||||
cache-from: type=gha,scope=stirling-unoserver
|
||||
cache-to: type=gha,mode=max,scope=stirling-unoserver
|
||||
platforms: linux/amd64
|
||||
tags: stirling-unoserver:pr-test
|
||||
provenance: false
|
||||
sbom: false
|
||||
|
||||
- name: Smoke test the built image
|
||||
run: |
|
||||
set -eu
|
||||
docker run -d --name unoserver-smoke \
|
||||
-e UNOSERVER_RECYCLE_INTERVAL_SECONDS=0 \
|
||||
stirling-unoserver:pr-test
|
||||
deadline=$((SECONDS + 60))
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
status=$(docker inspect -f '{{.State.Health.Status}}' unoserver-smoke 2>/dev/null || echo "starting")
|
||||
if [ "$status" = "healthy" ]; then
|
||||
echo "unoserver became healthy"
|
||||
docker logs unoserver-smoke | tail -30
|
||||
docker rm -f unoserver-smoke
|
||||
exit 0
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo "unoserver did not become healthy in time"
|
||||
docker logs unoserver-smoke || true
|
||||
docker rm -f unoserver-smoke || true
|
||||
exit 1
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
frontend: ${{ steps.changes.outputs.frontend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -186,7 +186,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ exampleYmlFiles/stirling/
|
||||
/testing/cucumber/junit/
|
||||
/testing/cucumber/report.html
|
||||
/testing/.failed_tests
|
||||
/.test-state/
|
||||
SwaggerDoc.json
|
||||
|
||||
# Runtime storage for uploaded files and user data (not Java source code)
|
||||
|
||||
+21
-48
@@ -1,5 +1,8 @@
|
||||
version: '3'
|
||||
|
||||
# All gradle invocations go through `bash gradlew` (the sh wrapper) so they work
|
||||
# uniformly on Linux/macOS (native bash) and Windows-with-Git-Bash.
|
||||
|
||||
tasks:
|
||||
dev:
|
||||
desc: "Start backend dev server"
|
||||
@@ -10,58 +13,43 @@ tasks:
|
||||
env:
|
||||
SERVER_PORT: '{{.PORT}}'
|
||||
cmds:
|
||||
- cmd: '{{if .AIENGINE_URL}}set "AIENGINE_URL={{.AIENGINE_URL}}" && set "AIENGINE_ENABLED=true" && {{end}}cmd /c gradlew.bat :stirling-pdf:bootRun'
|
||||
platforms: [windows]
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}./gradlew :stirling-pdf:bootRun'
|
||||
platforms: [linux, darwin]
|
||||
- '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}bash gradlew :stirling-pdf:bootRun'
|
||||
|
||||
dev:bundled:
|
||||
desc: "Clean + bootRun with frontend bundled into the backend (single :8080 server)"
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash gradlew clean bootRun -PbuildWithFrontend=true
|
||||
|
||||
build:
|
||||
desc: "Full backend build"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat clean build
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew clean build
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew clean build
|
||||
|
||||
build:fast:
|
||||
desc: "Build without tests"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat clean build -x test
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew clean build -x test
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew clean build -x test
|
||||
|
||||
build:ci:
|
||||
desc: "Build for CI (formatting checked separately)"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat build -PnoSpotless
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew build -PnoSpotless
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew build -PnoSpotless
|
||||
|
||||
test:
|
||||
desc: "Run backend tests"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat test
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew test
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew test
|
||||
|
||||
format:
|
||||
desc: "Auto-fix code formatting"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat spotlessApply
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew spotlessApply
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew spotlessApply
|
||||
|
||||
format:check:
|
||||
desc: "Check code formatting"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat spotlessCheck
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew spotlessCheck
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew spotlessCheck
|
||||
|
||||
fix:
|
||||
desc: "Auto-fix backend"
|
||||
@@ -71,10 +59,7 @@ tasks:
|
||||
swagger:
|
||||
desc: "Generate OpenAPI docs"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat :stirling-pdf:copySwaggerDoc
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew :stirling-pdf:copySwaggerDoc
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew :stirling-pdf:copySwaggerDoc
|
||||
sources:
|
||||
- app/core/src/main/java/**/*.java
|
||||
- app/proprietary/src/main/java/**/*.java
|
||||
@@ -92,31 +77,19 @@ tasks:
|
||||
desc: "Print project version"
|
||||
silent: true
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat printVersion --quiet | tail -1
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew printVersion --quiet | tail -1
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew printVersion --quiet | tail -1
|
||||
|
||||
licenses:check:
|
||||
desc: "Check dependency licenses"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat checkLicense --no-parallel
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew checkLicense --no-parallel
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew checkLicense --no-parallel
|
||||
|
||||
licenses:generate:
|
||||
desc: "Check and generate dependency license report"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat checkLicense generateLicenseReport --no-parallel
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew checkLicense generateLicenseReport --no-parallel
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew checkLicense generateLicenseReport --no-parallel
|
||||
|
||||
clean:
|
||||
desc: "Clean build artifacts"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat clean
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew clean
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew clean
|
||||
|
||||
@@ -1,12 +1,7 @@
|
||||
version: '3'
|
||||
|
||||
vars:
|
||||
# Single source of truth for the jlink module list (also read by
|
||||
# frontend/scripts/build-universal-mac-jre.sh during CI). The desktop
|
||||
# taskfile runs with dir: frontend (see Taskfile.yml), so this path
|
||||
# is relative to that directory.
|
||||
JLINK_MODULES:
|
||||
sh: tr -d '\n' < src-tauri/jlink-modules.txt
|
||||
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported"
|
||||
|
||||
tasks:
|
||||
prepare:
|
||||
|
||||
@@ -55,3 +55,9 @@ tasks:
|
||||
desc: "Tail docker compose logs"
|
||||
cmds:
|
||||
- docker compose -f {{.COMPOSE_DIR}}/docker-compose.yml logs -f
|
||||
|
||||
test:
|
||||
desc: "Run full Docker integration test suite (builds all variants and tests them)"
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/test.sh {{.CLI_ARGS}}
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
version: '3'
|
||||
|
||||
tasks:
|
||||
install:
|
||||
desc: "Install Playwright browsers"
|
||||
dir: frontend
|
||||
deps: [ ':frontend:install' ]
|
||||
cmds:
|
||||
- npx playwright install {{.CLI_ARGS}} --with-deps
|
||||
|
||||
stubbed:
|
||||
desc: "Run stubbed E2E tests"
|
||||
dir: frontend
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=stubbed {{.CLI_ARGS}}
|
||||
|
||||
live:
|
||||
desc: "Run live E2E tests"
|
||||
summary: |
|
||||
Auto-spawns a Spring Boot backend with isolated state under
|
||||
.test-state/playwright/ (purged on every run) so the suite doesn't
|
||||
touch your local DB, settings.yml, backups or customFiles. The
|
||||
backend is stopped automatically when the runner exits.
|
||||
|
||||
Pass extra Playwright flags via -- :
|
||||
task e2e:live -- --headed --grep merge
|
||||
deps:
|
||||
- live:backend
|
||||
- live:runner
|
||||
|
||||
live:backend:
|
||||
internal: true
|
||||
ignore_error: true
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
env:
|
||||
STIRLING_BASE_PATH: '{{.BASE_DIR}}'
|
||||
# Suppress the analytics opt-in modal that fires on first admin login.
|
||||
# It renders a Mantine overlay that intercepts pointer events and blocks
|
||||
# FirstLoginSlide from rendering, so the bootstrap spec times out waiting
|
||||
# for the password-change prompt.
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
# NOTE: SECURITY_INITIALLOGIN_USERNAME/PASSWORD are intentionally NOT set.
|
||||
# The live-setup project's bootstrap spec performs the real first-login
|
||||
# flow against the backend's default admin/stirling user, exercising the
|
||||
# forced-password-change UI and leaving the DB at admin/adminadmin for
|
||||
# the rest of the live suite. This is both real coverage of the first-
|
||||
# login flow and a stronger seed than env-var-driven user creation.
|
||||
cmds:
|
||||
# Wrapped in `bash -c` because Task's embedded shell (mvdan/sh) doesn't
|
||||
# support `&` + `$!` + `wait` reliably (gradle was never actually
|
||||
# backgrounded, so `$!` recorded a stale PID and the runner saw an
|
||||
# immediate "backend died"). Real bash backgrounds gradle properly and
|
||||
# gives us a stable PID to clean up.
|
||||
- |
|
||||
bash -c '
|
||||
set -e
|
||||
rm -rf "{{.BASE_DIR}}"
|
||||
mkdir -p "{{.BASE_DIR}}"
|
||||
# Background gradle and record its PID so the runner can clean up
|
||||
# the exact process tree (wrapper + forked Spring Boot JVM) without
|
||||
# resorting to fuzzy `pkill -f` patterns. `wait` keeps this script
|
||||
# alive for the lifetime of gradle so Task'"'"'s parallel deps stay
|
||||
# synchronised.
|
||||
bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 &
|
||||
GRADLE_PID=$!
|
||||
echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid"
|
||||
wait $GRADLE_PID
|
||||
'
|
||||
|
||||
live:runner:
|
||||
internal: true
|
||||
deps: [ ':frontend:prepare' ]
|
||||
dir: frontend
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
cmds:
|
||||
# Wrapped in `bash -c` because Task's embedded shell (mvdan/sh) only
|
||||
# supports `EXIT`/`ERR` in `trap`, not `INT`/`TERM`. CLI_ARGS are
|
||||
# passed positionally so quoted args like --grep "foo bar" survive.
|
||||
- |
|
||||
bash -c '
|
||||
set +e
|
||||
# bootRun forks a separate Spring Boot JVM as a child of the gradle
|
||||
# wrapper; killing only the wrapper leaves that JVM orphaned holding
|
||||
# :8080. Walk the recorded PID'"'"'s process tree via pgrep so we kill
|
||||
# every descendant, then lsof as a last-resort backstop.
|
||||
kill_tree() {
|
||||
local pid=$1
|
||||
for child in $(pgrep -P "$pid" 2>/dev/null); do
|
||||
kill_tree "$child"
|
||||
done
|
||||
kill "$pid" 2>/dev/null || true
|
||||
}
|
||||
cleanup() {
|
||||
if [ -f "{{.BASE_DIR}}/backend.pid" ]; then
|
||||
echo "Stopping backend..."
|
||||
kill_tree "$(cat "{{.BASE_DIR}}/backend.pid")"
|
||||
sleep 1
|
||||
lsof -ti:8080 2>/dev/null | xargs kill 2>/dev/null || true
|
||||
echo "Backend stopped"
|
||||
fi
|
||||
}
|
||||
# Trap so cleanup runs on every exit path: tests pass, tests fail,
|
||||
# backend never starts, user Ctrl-Cs. Without this the gradle JVM
|
||||
# is left orphaned holding :8080 whenever the wait loop times out.
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
echo "Waiting for backend on :8080 (up to 10min; first compile can be slow)..."
|
||||
WAITED=0
|
||||
while [ $WAITED -lt 600 ]; do
|
||||
if curl -sf http://localhost:8080/api/v1/info/status 2>/dev/null | grep -q UP; then
|
||||
echo "Backend ready, starting Playwright"
|
||||
break
|
||||
fi
|
||||
# Bail early if the backend already died: no point waiting 10min.
|
||||
if [ -f "{{.BASE_DIR}}/backend.pid" ]; then
|
||||
BACKEND_PID=$(cat "{{.BASE_DIR}}/backend.pid")
|
||||
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
|
||||
echo "Backend process exited before becoming ready; last 100 log lines:"
|
||||
tail -100 "{{.BASE_DIR}}/backend.log" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
WAITED=$((WAITED + 2))
|
||||
done
|
||||
if [ $WAITED -ge 600 ]; then
|
||||
echo "Backend did not become ready within 10min, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
npx playwright test --project=live "$@"
|
||||
' bash {{.CLI_ARGS}}
|
||||
|
||||
enterprise:
|
||||
desc: "Run enterprise E2E tests"
|
||||
summary: |
|
||||
Requires an already-running keycloak compose stack on :8080. Bring one
|
||||
up first with:
|
||||
task e2e:oauth:up
|
||||
task e2e:saml:up
|
||||
dir: frontend
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=enterprise {{.CLI_ARGS}}
|
||||
|
||||
cross-browser:
|
||||
desc: "Run stubbed E2E tests on Chromium, Firefox, and WebKit"
|
||||
dir: frontend
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=stubbed --project=stubbed-firefox --project=stubbed-webkit {{.CLI_ARGS}}
|
||||
|
||||
check:
|
||||
desc: "Run all E2E tests not requiring Docker"
|
||||
cmds:
|
||||
- task: stubbed
|
||||
- task: live
|
||||
|
||||
check:all:
|
||||
desc: "Run all E2E tests"
|
||||
summary: |
|
||||
Includes the enterprise project, which requires a keycloak compose
|
||||
stack on :8080. Bring one up first with:
|
||||
task e2e:oauth:up
|
||||
task e2e:saml:up
|
||||
cmds:
|
||||
- task: stubbed
|
||||
- task: live
|
||||
- task: enterprise
|
||||
|
||||
oauth:up:
|
||||
desc: "Start the OAuth keycloak test environment"
|
||||
summary: |
|
||||
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
|
||||
task e2e:oauth:up LICENSE_KEY=abc123
|
||||
Pass extra flags via -- :
|
||||
task e2e:oauth:up -- --auto --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-oauth-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
oauth:down:
|
||||
desc: "Stop the OAuth keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-oauth.yml down -v
|
||||
|
||||
saml:up:
|
||||
desc: "Start the SAML keycloak test environment"
|
||||
summary: |
|
||||
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
|
||||
task e2e:saml:up LICENSE_KEY=abc123
|
||||
Pass extra flags via -- :
|
||||
task e2e:saml:up -- --auto --with-storage --nobuild --language sv-SE
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-saml-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
saml:down:
|
||||
desc: "Stop the SAML keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-saml.yml down -v
|
||||
@@ -75,7 +75,7 @@ tasks:
|
||||
desc: "Run type checking"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- uv run pyright . --warnings
|
||||
- uv run pyrefly check
|
||||
|
||||
test:
|
||||
desc: "Run tests"
|
||||
|
||||
@@ -284,18 +284,6 @@ tasks:
|
||||
cmds:
|
||||
- npx vitest --coverage
|
||||
|
||||
test:e2e:
|
||||
desc: "Run E2E tests"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx playwright test {{.CLI_ARGS}}
|
||||
|
||||
test:e2e:install:
|
||||
desc: "Install E2E test browsers"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx playwright install {{.CLI_ARGS}} --with-deps
|
||||
|
||||
# ============================================================
|
||||
# Code Generation
|
||||
# ============================================================
|
||||
|
||||
@@ -22,6 +22,9 @@ includes:
|
||||
desktop:
|
||||
taskfile: .taskfiles/desktop.yml
|
||||
dir: frontend
|
||||
e2e:
|
||||
taskfile: .taskfiles/e2e.yml
|
||||
dir: .
|
||||
|
||||
tasks:
|
||||
# ============================================================
|
||||
|
||||
+12
@@ -59,6 +59,18 @@ public class InstallationPathConfig {
|
||||
}
|
||||
|
||||
private static String initializeBasePath() {
|
||||
// Allow tests / harnesses to redirect the entire state tree (configs,
|
||||
// backups, customFiles, pipeline, logs) to an isolated location via
|
||||
// -Dstirling.base-path=... or STIRLING_BASE_PATH=... so a Playwright
|
||||
// run never touches a developer's working state.
|
||||
String override = System.getProperty("stirling.base-path");
|
||||
if (override == null || override.isBlank()) {
|
||||
override = System.getenv("STIRLING_BASE_PATH");
|
||||
}
|
||||
if (override != null && !override.isBlank()) {
|
||||
boolean hasTrailingSeparator = override.endsWith("/") || override.endsWith("\\");
|
||||
return hasTrailingSeparator ? override : override + File.separator;
|
||||
}
|
||||
return "." + File.separator;
|
||||
}
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
|
||||
@Bean
|
||||
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
|
||||
@@ -246,6 +247,19 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP timeouts for loopback calls to internal Stirling API endpoints, used by the AI workflow
|
||||
* executor and the pipeline processor. A bounded read timeout prevents a hung tool (e.g. an
|
||||
* infinite loop in a PDF processing service) from stalling the entire chat workflow forever.
|
||||
* Tools that legitimately need longer than the read timeout should be invoked through the async
|
||||
* job executor instead of synchronously.
|
||||
*/
|
||||
@Data
|
||||
public static class InternalApi {
|
||||
private int connectTimeoutSeconds = 10;
|
||||
private int readTimeoutSeconds = 300;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Legal {
|
||||
private String termsAndConditions;
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.common.model.api.general;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode
|
||||
public class EditTextOperation {
|
||||
|
||||
@Schema(description = "The literal text to find.", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String find;
|
||||
|
||||
@Schema(
|
||||
description = "The replacement text. May be empty to delete the matched text.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String replace;
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import java.io.UncheckedIOException;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -12,15 +13,18 @@ import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RequestCallback;
|
||||
import org.springframework.web.client.ResourceAccessException;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import jakarta.servlet.ServletContext;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
@@ -50,16 +54,29 @@ public class InternalApiClient {
|
||||
private final UserServiceInterface userService;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final Environment environment;
|
||||
private final Duration readTimeout;
|
||||
private final RestTemplate restTemplate;
|
||||
|
||||
public InternalApiClient(
|
||||
ServletContext servletContext,
|
||||
@Autowired(required = false) UserServiceInterface userService,
|
||||
TempFileManager tempFileManager,
|
||||
Environment environment) {
|
||||
Environment environment,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.servletContext = servletContext;
|
||||
this.userService = userService;
|
||||
this.tempFileManager = tempFileManager;
|
||||
this.environment = environment;
|
||||
ApplicationProperties.InternalApi internalApi = applicationProperties.getInternalApi();
|
||||
// A bounded read timeout is what protects the workflow when an internal tool hangs
|
||||
// (e.g. an infinite loop in a PDF processing service). The connect timeout is short
|
||||
// because this is a loopback call; if connecting takes longer than a few seconds the
|
||||
// local server is itself unhealthy.
|
||||
this.readTimeout = Duration.ofSeconds(internalApi.getReadTimeoutSeconds());
|
||||
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
|
||||
factory.setConnectTimeout(Duration.ofSeconds(internalApi.getConnectTimeoutSeconds()));
|
||||
factory.setReadTimeout(readTimeout);
|
||||
this.restTemplate = new RestTemplate(factory);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -74,7 +91,6 @@ public class InternalApiClient {
|
||||
validateUrl(endpointPath);
|
||||
String url = getBaseUrl() + endpointPath;
|
||||
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
String apiKey = getApiKeyForUser();
|
||||
if (apiKey != null && !apiKey.isEmpty()) {
|
||||
@@ -84,26 +100,38 @@ public class InternalApiClient {
|
||||
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
|
||||
RequestCallback requestCallback = restTemplate.httpEntityCallback(entity, Resource.class);
|
||||
|
||||
return restTemplate.execute(
|
||||
url,
|
||||
HttpMethod.POST,
|
||||
requestCallback,
|
||||
response -> {
|
||||
try {
|
||||
TempFile tempFile = tempFileManager.createManagedTempFile("internal-api");
|
||||
Files.copy(
|
||||
response.getBody(),
|
||||
tempFile.getPath(),
|
||||
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
String filename = extractFilename(response.getHeaders());
|
||||
TempFileResource resource = new TempFileResource(tempFile, filename);
|
||||
return ResponseEntity.status(response.getStatusCode())
|
||||
.headers(response.getHeaders())
|
||||
.body(resource);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
try {
|
||||
return restTemplate.execute(
|
||||
url,
|
||||
HttpMethod.POST,
|
||||
requestCallback,
|
||||
response -> {
|
||||
try {
|
||||
TempFile tempFile =
|
||||
tempFileManager.createManagedTempFile("internal-api");
|
||||
Files.copy(
|
||||
response.getBody(),
|
||||
tempFile.getPath(),
|
||||
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
String filename = extractFilename(response.getHeaders());
|
||||
TempFileResource resource = new TempFileResource(tempFile, filename);
|
||||
return ResponseEntity.status(response.getStatusCode())
|
||||
.headers(response.getHeaders())
|
||||
.body(resource);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
} catch (ResourceAccessException e) {
|
||||
// RestTemplate wraps low-level I/O failures in ResourceAccessException. Only the
|
||||
// SocketTimeoutException-rooted case is a real timeout; other I/O failures (connection
|
||||
// refused, DNS, etc.) propagate as-is so the upstream generic handler can describe
|
||||
// them accurately.
|
||||
if (e.getCause() instanceof java.net.SocketTimeoutException) {
|
||||
throw new InternalApiTimeoutException(endpointPath, readTimeout, e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Thrown when an internal Stirling tool invocation exceeds its configured read timeout or otherwise
|
||||
* fails at the transport layer. Distinguishes a hung/timed-out tool from a tool that returned a
|
||||
* non-2xx HTTP response (which is reported via the response status itself), so callers can present
|
||||
* a clear "the tool didn't respond in time" message to the user instead of a generic stack trace.
|
||||
*/
|
||||
public class InternalApiTimeoutException extends RuntimeException {
|
||||
|
||||
private final String endpointPath;
|
||||
private final Duration readTimeout;
|
||||
|
||||
public InternalApiTimeoutException(String endpointPath, Duration readTimeout, Throwable cause) {
|
||||
super(buildMessage(endpointPath, readTimeout, cause), cause);
|
||||
this.endpointPath = endpointPath;
|
||||
this.readTimeout = readTimeout;
|
||||
}
|
||||
|
||||
public String getEndpointPath() {
|
||||
return endpointPath;
|
||||
}
|
||||
|
||||
public Duration getReadTimeout() {
|
||||
return readTimeout;
|
||||
}
|
||||
|
||||
private static String buildMessage(String endpointPath, Duration readTimeout, Throwable cause) {
|
||||
String reason = cause != null && cause.getMessage() != null ? cause.getMessage() : "";
|
||||
return String.format(
|
||||
"Internal tool %s did not respond within %ds (%s)",
|
||||
endpointPath, readTimeout.toSeconds(), reason);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import io.github.pixee.security.BoundedLineReader;
|
||||
|
||||
@@ -202,7 +203,15 @@ public class ProcessExecutor {
|
||||
boolean useSemaphore = true;
|
||||
List<String> commandToRun = command;
|
||||
if (shouldUseUnoServerPool(command)) {
|
||||
unoLease = unoServerPool.acquireEndpoint();
|
||||
try {
|
||||
unoLease = unoServerPool.acquireEndpoint(timeoutDuration, TimeUnit.MINUTES);
|
||||
} catch (TimeoutException e) {
|
||||
throw new IOException(
|
||||
"All unoserver endpoints busy; request timed out after "
|
||||
+ timeoutDuration
|
||||
+ " minutes",
|
||||
e);
|
||||
}
|
||||
commandToRun = applyUnoServerEndpoint(command, unoLease.getEndpoint());
|
||||
useSemaphore = false;
|
||||
}
|
||||
|
||||
@@ -3,8 +3,11 @@ package stirling.software.common.util;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
@@ -42,6 +45,27 @@ public class UnoServerPool {
|
||||
return new UnoServerLease(endpoints.get(index), index, this);
|
||||
}
|
||||
|
||||
/** Fail-fast variant; non-positive timeout falls back to unbounded acquire. */
|
||||
public UnoServerLease acquireEndpoint(long timeout, TimeUnit unit)
|
||||
throws InterruptedException, TimeoutException {
|
||||
if (endpoints.isEmpty()) {
|
||||
return new UnoServerLease(defaultEndpoint(), null, this);
|
||||
}
|
||||
if (timeout <= 0) {
|
||||
return acquireEndpoint();
|
||||
}
|
||||
|
||||
Integer index = availableIndices.poll(timeout, unit);
|
||||
if (index == null) {
|
||||
throw new TimeoutException(
|
||||
"Timed out waiting for a free unoserver endpoint after "
|
||||
+ timeout
|
||||
+ " "
|
||||
+ unit.name().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
return new UnoServerLease(endpoints.get(index), index, this);
|
||||
}
|
||||
|
||||
private void releaseEndpoint(Integer index) {
|
||||
if (index != null) {
|
||||
availableIndices.offer(index);
|
||||
|
||||
+18
-6
@@ -6,15 +6,18 @@ import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.JavaType;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
|
||||
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
|
||||
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
|
||||
*/
|
||||
@Slf4j
|
||||
public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
|
||||
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private final ObjectMapper objectMapper =
|
||||
JsonMapper.builder()
|
||||
@@ -22,6 +25,12 @@ public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final Class<T> elementType;
|
||||
|
||||
public StringToArrayListPropertyEditor(Class<T> elementType) {
|
||||
this.elementType = elementType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
@@ -29,8 +38,11 @@ public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
TypeReference<ArrayList<RedactionArea>> typeRef = new TypeReference<>() {};
|
||||
List<RedactionArea> list = objectMapper.readValue(text, typeRef);
|
||||
JavaType listType =
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(ArrayList.class, elementType);
|
||||
List<T> list = objectMapper.readValue(text, listType);
|
||||
setValue(list);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception while converting {}", e);
|
||||
|
||||
+87
-3
@@ -22,11 +22,13 @@ import org.springframework.mock.env.MockEnvironment;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RequestCallback;
|
||||
import org.springframework.web.client.ResourceAccessException;
|
||||
import org.springframework.web.client.ResponseExtractor;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import jakarta.servlet.ServletContext;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@@ -42,8 +44,19 @@ class InternalApiClientTest {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(servletContext.getContextPath()).thenReturn("");
|
||||
client = newClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a fresh client. Tests that use {@link org.mockito.Mockito#mockConstruction} to
|
||||
* intercept {@link RestTemplate} must call this from inside their {@code mockConstruction}
|
||||
* block, since the client now caches one RestTemplate per instance at construction time.
|
||||
*/
|
||||
private InternalApiClient newClient() {
|
||||
MockEnvironment environment = new MockEnvironment().withProperty("server.port", "8080");
|
||||
client = new InternalApiClient(servletContext, userService, tempFileManager, environment);
|
||||
ApplicationProperties applicationProperties = new ApplicationProperties();
|
||||
return new InternalApiClient(
|
||||
servletContext, userService, tempFileManager, environment, applicationProperties);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -75,7 +88,10 @@ class InternalApiClientTest {
|
||||
.thenAnswer(inv -> fakeOkResponse(inv.getArgument(3)));
|
||||
})) {
|
||||
|
||||
ResponseEntity<Resource> response = client.post("/api/v1/general/merge-pdfs", body);
|
||||
// Reconstruct the client so its cached RestTemplate is the mocked one.
|
||||
InternalApiClient mockedClient = newClient();
|
||||
ResponseEntity<Resource> response =
|
||||
mockedClient.post("/api/v1/general/merge-pdfs", body);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
@@ -86,6 +102,72 @@ class InternalApiClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postWrapsSocketTimeoutAsInternalApiTimeoutException() {
|
||||
// Simulates the read-timeout case: RestTemplate wraps SocketTimeoutException in
|
||||
// ResourceAccessException when the underlying socket times out. The client must repackage
|
||||
// that into a typed timeout exception that carries the failing endpoint and the configured
|
||||
// read timeout, so the workflow layer can surface a clean "tool didn't respond" message
|
||||
// to the user.
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("fileInput", namedResource("input.pdf", "data"));
|
||||
|
||||
try (var ignored =
|
||||
mockConstruction(
|
||||
RestTemplate.class,
|
||||
(rt, ctx) -> {
|
||||
when(rt.httpEntityCallback(any(), eq(Resource.class)))
|
||||
.thenAnswer(inv -> (RequestCallback) req -> {});
|
||||
when(rt.execute(anyString(), eq(HttpMethod.POST), any(), any()))
|
||||
.thenThrow(
|
||||
new ResourceAccessException(
|
||||
"I/O error on POST request: Read timed out",
|
||||
new java.net.SocketTimeoutException(
|
||||
"Read timed out")));
|
||||
})) {
|
||||
|
||||
InternalApiClient mockedClient = newClient();
|
||||
InternalApiTimeoutException thrown =
|
||||
assertThrows(
|
||||
InternalApiTimeoutException.class,
|
||||
() -> mockedClient.post("/api/v1/general/merge-pdfs", body));
|
||||
|
||||
assertEquals("/api/v1/general/merge-pdfs", thrown.getEndpointPath());
|
||||
assertNotNull(thrown.getReadTimeout());
|
||||
assertTrue(thrown.getMessage().contains("/api/v1/general/merge-pdfs"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postRethrowsNonTimeoutResourceAccessExceptionAsIs() {
|
||||
// ResourceAccessException covers more than just timeouts (e.g. connection refused, DNS
|
||||
// failure). Only SocketTimeoutException-rooted failures are timeouts; everything else
|
||||
// must propagate so the upstream generic handler can label it correctly instead of lying
|
||||
// about a "tool didn't respond" timeout.
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("fileInput", namedResource("input.pdf", "data"));
|
||||
|
||||
try (var ignored =
|
||||
mockConstruction(
|
||||
RestTemplate.class,
|
||||
(rt, ctx) -> {
|
||||
when(rt.httpEntityCallback(any(), eq(Resource.class)))
|
||||
.thenAnswer(inv -> (RequestCallback) req -> {});
|
||||
when(rt.execute(anyString(), eq(HttpMethod.POST), any(), any()))
|
||||
.thenThrow(
|
||||
new ResourceAccessException(
|
||||
"I/O error on POST request: Connection refused",
|
||||
new java.net.ConnectException(
|
||||
"Connection refused")));
|
||||
})) {
|
||||
|
||||
InternalApiClient mockedClient = newClient();
|
||||
assertThrows(
|
||||
ResourceAccessException.class,
|
||||
() -> mockedClient.post("/api/v1/general/merge-pdfs", body));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postRejectsDisallowedPath() {
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
@@ -125,8 +207,10 @@ class InternalApiClientTest {
|
||||
.thenAnswer(inv -> fakeOkResponse(inv.getArgument(3)));
|
||||
})) {
|
||||
|
||||
// Reconstruct the client so its cached RestTemplate is the mocked one.
|
||||
InternalApiClient mockedClient = newClient();
|
||||
ResponseEntity<Resource> response =
|
||||
client.post("/api/v1/ai/tools/pdf-comment-agent", body);
|
||||
mockedClient.post("/api/v1/ai/tools/pdf-comment-agent", body);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
|
||||
@@ -9,6 +9,7 @@ import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -173,6 +174,70 @@ public class UnoServerPoolTest {
|
||||
assertEquals(port1, port2, "Should reuse the same endpoint after release");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAcquireWithTimeoutFailsFast() throws InterruptedException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
|
||||
createEndpoints(1);
|
||||
UnoServerPool pool = new UnoServerPool(endpoints);
|
||||
|
||||
UnoServerPool.UnoServerLease held = pool.acquireEndpoint();
|
||||
|
||||
long start = System.nanoTime();
|
||||
assertThrows(
|
||||
TimeoutException.class,
|
||||
() -> pool.acquireEndpoint(100, TimeUnit.MILLISECONDS),
|
||||
"Should time out when no endpoint available");
|
||||
long elapsedMs = (System.nanoTime() - start) / 1_000_000;
|
||||
|
||||
assertTrue(
|
||||
elapsedMs >= 100 && elapsedMs < 1000,
|
||||
"Should fail-fast after roughly the timeout, got " + elapsedMs + "ms");
|
||||
|
||||
held.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAcquireWithTimeoutSucceedsWhenAvailable()
|
||||
throws InterruptedException, TimeoutException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
|
||||
createEndpoints(1);
|
||||
UnoServerPool pool = new UnoServerPool(endpoints);
|
||||
|
||||
try (UnoServerPool.UnoServerLease lease = pool.acquireEndpoint(1, TimeUnit.SECONDS)) {
|
||||
assertNotNull(lease);
|
||||
assertEquals(2003, lease.getEndpoint().getPort());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAcquireWithZeroTimeoutBlocksUnbounded() throws InterruptedException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
|
||||
createEndpoints(1);
|
||||
UnoServerPool pool = new UnoServerPool(endpoints);
|
||||
UnoServerPool.UnoServerLease held = pool.acquireEndpoint();
|
||||
|
||||
AtomicInteger acquired = new AtomicInteger(0);
|
||||
Thread t =
|
||||
Thread.ofVirtual()
|
||||
.start(
|
||||
() -> {
|
||||
try {
|
||||
UnoServerPool.UnoServerLease lease =
|
||||
pool.acquireEndpoint(0, TimeUnit.MILLISECONDS);
|
||||
acquired.incrementAndGet();
|
||||
lease.close();
|
||||
} catch (Exception e) {
|
||||
fail("unexpected: " + e);
|
||||
}
|
||||
});
|
||||
|
||||
Thread.sleep(150);
|
||||
assertEquals(0, acquired.get(), "Should still be blocked");
|
||||
held.close();
|
||||
t.join(1000);
|
||||
assertEquals(1, acquired.get(), "Should acquire after release");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testHostLocationAndProtocol() throws InterruptedException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints = new ArrayList<>();
|
||||
|
||||
+2
-2
@@ -11,11 +11,11 @@ import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
class StringToArrayListPropertyEditorTest {
|
||||
|
||||
private StringToArrayListPropertyEditor editor;
|
||||
private StringToArrayListPropertyEditor<RedactionArea> editor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
editor = new StringToArrayListPropertyEditor();
|
||||
editor = new StringToArrayListPropertyEditor<>(RedactionArea.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -65,7 +65,7 @@ dependencies {
|
||||
exclude group: 'com.fasterxml.jackson.jaxrs'
|
||||
exclude group: 'com.fasterxml.jackson.module', module: 'jackson-module-jaxb-annotations'
|
||||
}
|
||||
implementation 'commons-io:commons-io:2.21.0'
|
||||
implementation 'commons-io:commons-io:2.22.0'
|
||||
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
|
||||
implementation "org.bouncycastle:bcpkix-jdk18on:$bouncycastleVersion"
|
||||
implementation 'io.micrometer:micrometer-core'
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.WebDataBinder;
|
||||
import org.springframework.web.bind.annotation.InitBinder;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.swagger.StandardPdfResponse;
|
||||
import stirling.software.SPDF.model.api.general.EditTextRequest;
|
||||
import stirling.software.SPDF.model.json.PdfJsonDocument;
|
||||
import stirling.software.SPDF.model.json.PdfJsonPage;
|
||||
import stirling.software.SPDF.model.json.PdfJsonTextElement;
|
||||
import stirling.software.SPDF.service.PdfJsonConversionService;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor;
|
||||
|
||||
/**
|
||||
* Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the
|
||||
* input PDF is parsed into the editable JSON model, find/replace operations are applied to the
|
||||
* {@code text} field of each text element, and the mutated model is rebuilt into a PDF.
|
||||
*
|
||||
* <p>Matching joins all text elements on a page into a single string before searching, so find
|
||||
* strings can span multiple visual runs (titles split per word, kerning-broken phrases, etc.).
|
||||
*
|
||||
* <p>For cross-element matches the entire replacement is written into the first matched element,
|
||||
* any intermediate elements are emptied, and the suffix of the last matched element is preserved.
|
||||
* This anchors the new text at the leftmost match position and lets the font lay it out as one run,
|
||||
* which is more visually reliable than redistributing words to the original per-element X positions
|
||||
* (those positions are calibrated to the original word widths and rarely match the replacement
|
||||
* widths). Centered or tracked layouts will become left-aligned at the original first-word position
|
||||
* when their text is replaced - the trade-off is accepted in favour of avoiding glyph overlaps and
|
||||
* gaps from variable-width word distribution.
|
||||
*/
|
||||
@Slf4j
|
||||
@GeneralApi
|
||||
@RequiredArgsConstructor
|
||||
public class EditTextController {
|
||||
|
||||
private static final Pattern FILE_EXTENSION_PATTERN = Pattern.compile("[.][^.]+$");
|
||||
|
||||
private final PdfJsonConversionService pdfJsonConversionService;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
@InitBinder
|
||||
public void initBinder(WebDataBinder binder) {
|
||||
binder.registerCustomEditor(
|
||||
List.class,
|
||||
"edits",
|
||||
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = "multipart/form-data", value = "/edit-text")
|
||||
@StandardPdfResponse
|
||||
@Operation(
|
||||
summary = "Edit text in a PDF via find and replace",
|
||||
description =
|
||||
"Applies an ordered list of find/replace operations to the text in a PDF and"
|
||||
+ " returns the edited PDF. Useful for find-and-replace, bulk renames"
|
||||
+ " (e.g. updating a company name throughout a document), and copy"
|
||||
+ " editing where the AI agent has identified specific replacements."
|
||||
+ " Matching is performed against the joined text of each page, so"
|
||||
+ " find strings can span multiple visual runs (titles split per word,"
|
||||
+ " kerning-broken phrases). Cross-element matches are written as a"
|
||||
+ " single replacement run anchored at the leftmost matched position;"
|
||||
+ " centered or tracked text may shift left when its content changes."
|
||||
+ " Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<Resource> editText(@ModelAttribute EditTextRequest request)
|
||||
throws Exception {
|
||||
MultipartFile inputFile = request.getFileInput();
|
||||
if (inputFile == null) {
|
||||
throw ExceptionUtils.createFileNullOrEmptyException();
|
||||
}
|
||||
List<EditTextOperation> edits = request.getEdits();
|
||||
if (edits == null || edits.isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.editText.no.edits",
|
||||
"No find/replace operations provided for text editing");
|
||||
}
|
||||
for (EditTextOperation edit : edits) {
|
||||
if (edit == null || edit.getFind() == null || edit.getFind().isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.editText.empty.find", "Each edit must have a non-empty find string");
|
||||
}
|
||||
}
|
||||
|
||||
boolean wholeWordSearch = Boolean.TRUE.equals(request.getWholeWordSearch());
|
||||
List<CompiledEdit> compiledEdits = compileEdits(edits, wholeWordSearch);
|
||||
|
||||
PdfJsonDocument document = pdfJsonConversionService.convertPdfToJsonDocument(inputFile);
|
||||
Set<Integer> pageFilter = resolvePageFilter(request, document);
|
||||
int modifiedSpans = applyEdits(document, compiledEdits, pageFilter);
|
||||
log.info(
|
||||
"edit-text: modified {} text span(s) using {} edit(s) on {} page(s)",
|
||||
modifiedSpans,
|
||||
compiledEdits.size(),
|
||||
pageFilter == null
|
||||
? document.getPages() == null ? 0 : document.getPages().size()
|
||||
: pageFilter.size());
|
||||
|
||||
String docName = buildOutputFilename(inputFile);
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try (OutputStream os = Files.newOutputStream(tempOut.getPath())) {
|
||||
pdfJsonConversionService.convertJsonToPdf(document, os);
|
||||
} catch (Exception e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
return WebResponseUtils.pdfFileToWebResponse(tempOut, docName);
|
||||
}
|
||||
|
||||
private List<CompiledEdit> compileEdits(
|
||||
List<EditTextOperation> edits, boolean wholeWordSearch) {
|
||||
return edits.stream().map(edit -> compileEdit(edit, wholeWordSearch)).toList();
|
||||
}
|
||||
|
||||
private CompiledEdit compileEdit(EditTextOperation edit, boolean wholeWordSearch) {
|
||||
// Always treat the user-supplied find string as a literal: Pattern.quote escapes any
|
||||
// regex metacharacters, so the constructed pattern can only ever do a literal match
|
||||
// (optionally bounded by our own word-boundary anchors). This rules out catastrophic
|
||||
// backtracking from a malicious find string.
|
||||
String findRaw = edit.getFind();
|
||||
String replacement = Objects.toString(edit.getReplace(), "");
|
||||
String regex = Pattern.quote(findRaw);
|
||||
if (wholeWordSearch) {
|
||||
// Use lookarounds rather than \b so the bound works even when the find string starts
|
||||
// or ends with a non-word character (e.g. "-foo" or "$id"). \b only fires at a
|
||||
// word/non-word transition, which would never trigger for a find that starts with a
|
||||
// non-word char and was preceded by another non-word char in the source text.
|
||||
regex = "(?<!\\w)(?:" + regex + ")(?!\\w)";
|
||||
}
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
String safeReplacement = Matcher.quoteReplacement(replacement);
|
||||
return new CompiledEdit(pattern, safeReplacement);
|
||||
}
|
||||
|
||||
private Set<Integer> resolvePageFilter(EditTextRequest request, PdfJsonDocument document) {
|
||||
String pageNumbers = request.getPageNumbers();
|
||||
int totalPages = document.getPages() == null ? 0 : document.getPages().size();
|
||||
if (totalPages == 0) {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
if (pageNumbers == null || pageNumbers.isBlank() || "all".equalsIgnoreCase(pageNumbers)) {
|
||||
return null;
|
||||
}
|
||||
List<Integer> pages = GeneralUtils.parsePageList(pageNumbers, totalPages, true);
|
||||
return new HashSet<>(pages);
|
||||
}
|
||||
|
||||
private int applyEdits(
|
||||
PdfJsonDocument document, List<CompiledEdit> edits, Set<Integer> pageFilter) {
|
||||
List<PdfJsonPage> pages = document.getPages();
|
||||
if (pages == null) {
|
||||
return 0;
|
||||
}
|
||||
int modifiedSpans = 0;
|
||||
// The page filter is built from parsePageList against pages.size(), so it returns 1-based
|
||||
// positional page numbers. Match against the same positional numbering here rather than
|
||||
// the per-page pageNumber field, which can become non-sequential after split/merge round
|
||||
// trips and would misalign the filter.
|
||||
for (int i = 0; i < pages.size(); i++) {
|
||||
int positionalPageNumber = i + 1;
|
||||
if (pageFilter != null && !pageFilter.contains(positionalPageNumber)) {
|
||||
continue;
|
||||
}
|
||||
modifiedSpans += applyEditsToPage(pages.get(i), edits);
|
||||
}
|
||||
return modifiedSpans;
|
||||
}
|
||||
|
||||
private int applyEditsToPage(PdfJsonPage page, List<CompiledEdit> edits) {
|
||||
List<PdfJsonTextElement> elements = page.getTextElements();
|
||||
if (elements == null || elements.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
Set<Integer> modifiedIndices = new HashSet<>();
|
||||
for (CompiledEdit edit : edits) {
|
||||
applyEditToPage(elements, edit, modifiedIndices);
|
||||
}
|
||||
for (Integer index : modifiedIndices) {
|
||||
// Char codes were captured for the original glyph sequence; clear them so the rebuild
|
||||
// re-encodes from the new text via the font.
|
||||
elements.get(index).setCharCodes(null);
|
||||
}
|
||||
return modifiedIndices.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a single edit across the page by matching against the concatenation of all element
|
||||
* texts, then writing the replacement back into the originating element(s).
|
||||
*/
|
||||
private void applyEditToPage(
|
||||
List<PdfJsonTextElement> elements, CompiledEdit edit, Set<Integer> modifiedIndices) {
|
||||
StringBuilder joined = new StringBuilder();
|
||||
int[] starts = new int[elements.size()];
|
||||
int[] ends = new int[elements.size()];
|
||||
for (int i = 0; i < elements.size(); i++) {
|
||||
starts[i] = joined.length();
|
||||
String text = elements.get(i).getText();
|
||||
if (text != null) {
|
||||
joined.append(text);
|
||||
}
|
||||
ends[i] = joined.length();
|
||||
}
|
||||
|
||||
Matcher matcher = edit.pattern().matcher(joined);
|
||||
List<MatchSpan> spans = new ArrayList<>();
|
||||
StringBuffer interpolation = new StringBuffer();
|
||||
int previousAppendPosition = 0;
|
||||
while (matcher.find()) {
|
||||
if (matcher.start() == matcher.end()) {
|
||||
// Skip zero-length matches (e.g. /a*/ on empty input) — they cannot be applied.
|
||||
continue;
|
||||
}
|
||||
int sizeBefore = interpolation.length();
|
||||
matcher.appendReplacement(interpolation, edit.replacement());
|
||||
int prefixLength = matcher.start() - previousAppendPosition;
|
||||
String actualReplacement =
|
||||
interpolation.substring(sizeBefore + prefixLength, interpolation.length());
|
||||
spans.add(new MatchSpan(matcher.start(), matcher.end(), actualReplacement));
|
||||
previousAppendPosition = matcher.end();
|
||||
}
|
||||
|
||||
// Apply right-to-left so earlier match positions stay valid as we mutate elements.
|
||||
for (int i = spans.size() - 1; i >= 0; i--) {
|
||||
MatchSpan span = spans.get(i);
|
||||
int firstElement = findElementForCharIndex(starts, ends, span.start());
|
||||
int lastElement = findElementForCharIndex(starts, ends, span.end() - 1);
|
||||
if (firstElement < 0 || lastElement < 0) {
|
||||
continue;
|
||||
}
|
||||
applyMatchToElements(
|
||||
elements, starts, span, firstElement, lastElement, modifiedIndices);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the element whose text covers the character at {@code charIndex} in the joined string.
|
||||
* Returns -1 if no element covers that index (which should not happen for valid match spans).
|
||||
*/
|
||||
private static int findElementForCharIndex(int[] starts, int[] ends, int charIndex) {
|
||||
for (int i = 0; i < starts.length; i++) {
|
||||
if (starts[i] <= charIndex && charIndex < ends[i]) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static void applyMatchToElements(
|
||||
List<PdfJsonTextElement> elements,
|
||||
int[] starts,
|
||||
MatchSpan span,
|
||||
int firstElement,
|
||||
int lastElement,
|
||||
Set<Integer> modifiedIndices) {
|
||||
if (firstElement == lastElement) {
|
||||
PdfJsonTextElement element = elements.get(firstElement);
|
||||
String text = nullToEmpty(element.getText());
|
||||
int matchStartInElement = span.start() - starts[firstElement];
|
||||
int matchEndInElement = span.end() - starts[firstElement];
|
||||
element.setText(
|
||||
text.substring(0, matchStartInElement)
|
||||
+ span.replacement()
|
||||
+ text.substring(matchEndInElement));
|
||||
modifiedIndices.add(firstElement);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cross-element match: write the whole replacement into the first matched element, empty
|
||||
// any intermediate elements, and keep only the suffix of the last matched element. The
|
||||
// JSON->PDF rebuild concatenates per-token text, so the font lays out the replacement as
|
||||
// one continuous run anchored at the first element's X position.
|
||||
String firstText = nullToEmpty(elements.get(firstElement).getText());
|
||||
int firstSplit = span.start() - starts[firstElement];
|
||||
elements.get(firstElement).setText(firstText.substring(0, firstSplit) + span.replacement());
|
||||
modifiedIndices.add(firstElement);
|
||||
|
||||
for (int mid = firstElement + 1; mid < lastElement; mid++) {
|
||||
elements.get(mid).setText("");
|
||||
modifiedIndices.add(mid);
|
||||
}
|
||||
|
||||
String lastText = nullToEmpty(elements.get(lastElement).getText());
|
||||
int lastSplit = span.end() - starts[lastElement];
|
||||
elements.get(lastElement).setText(lastText.substring(lastSplit));
|
||||
modifiedIndices.add(lastElement);
|
||||
}
|
||||
|
||||
private static String nullToEmpty(String value) {
|
||||
return value != null ? value : "";
|
||||
}
|
||||
|
||||
private String buildOutputFilename(MultipartFile inputFile) {
|
||||
String originalName = inputFile.getOriginalFilename();
|
||||
String baseName =
|
||||
(originalName != null && !originalName.isBlank())
|
||||
? FILE_EXTENSION_PATTERN
|
||||
.matcher(Filenames.toSimpleFileName(originalName))
|
||||
.replaceFirst("")
|
||||
: "document";
|
||||
return baseName + "_edited.pdf";
|
||||
}
|
||||
|
||||
private record CompiledEdit(Pattern pattern, String replacement) {}
|
||||
|
||||
private record MatchSpan(int start, int end, String replacement) {}
|
||||
}
|
||||
+3
-1
@@ -96,7 +96,9 @@ public class RedactController {
|
||||
@InitBinder
|
||||
public void initBinder(WebDataBinder binder) {
|
||||
binder.registerCustomEditor(
|
||||
List.class, "redactions", new StringToArrayListPropertyEditor());
|
||||
List.class,
|
||||
"redactions",
|
||||
new StringToArrayListPropertyEditor<>(RedactionArea.class));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(value = "/redact", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
|
||||
@@ -682,7 +682,7 @@ public class GlobalExceptionHandler {
|
||||
*
|
||||
* @param ex the MaxUploadSizeExceededException
|
||||
* @param request the HTTP servlet request
|
||||
* @return ProblemDetail with HTTP 413 PAYLOAD_TOO_LARGE
|
||||
* @return ProblemDetail with HTTP 413 CONTENT_TOO_LARGE
|
||||
*/
|
||||
@ExceptionHandler(MaxUploadSizeExceededException.class)
|
||||
public ResponseEntity<ProblemDetail> handleMaxUploadSize(
|
||||
@@ -706,7 +706,7 @@ public class GlobalExceptionHandler {
|
||||
getLocalizedMessage("error.fileTooLarge.title", ErrorTitles.FILE_TOO_LARGE_DEFAULT);
|
||||
|
||||
ProblemDetail problemDetail =
|
||||
createBaseProblemDetail(HttpStatus.PAYLOAD_TOO_LARGE, message, request);
|
||||
createBaseProblemDetail(HttpStatus.CONTENT_TOO_LARGE, message, request);
|
||||
problemDetail.setType(URI.create(ErrorTypes.FILE_TOO_LARGE));
|
||||
problemDetail.setTitle(title);
|
||||
problemDetail.setProperty("title", title); // Ensure serialization
|
||||
@@ -724,7 +724,7 @@ public class GlobalExceptionHandler {
|
||||
problemDetail.setProperty(
|
||||
"actionRequired", "Reduce the file size to be within the upload limit.");
|
||||
|
||||
return ResponseEntity.status(HttpStatus.PAYLOAD_TOO_LARGE)
|
||||
return ResponseEntity.status(HttpStatus.CONTENT_TOO_LARGE)
|
||||
.contentType(PROBLEM_JSON)
|
||||
.body(problemDetail);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package stirling.software.SPDF.model.api.general;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.SPDF.model.api.PDFWithPageNums;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class EditTextRequest extends PDFWithPageNums {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Ordered list of find/replace operations. Each replaces every occurrence on"
|
||||
+ " the selected pages, in order; later operations see the result of"
|
||||
+ " earlier ones (so 'foo'->'foos' then 'foos'->'bars' turns 'foo'"
|
||||
+ " into 'bars').",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<EditTextOperation> edits;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Whether matches must be whole words (boundaries determined by non-word"
|
||||
+ " characters)",
|
||||
defaultValue = "false")
|
||||
private Boolean wholeWordSearch;
|
||||
}
|
||||
+118
-10
@@ -252,6 +252,17 @@ public class PdfJsonConversionService {
|
||||
convertPdfToJson(file, null, false, out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a PDF to the editable {@link PdfJsonDocument} model. Convenience wrapper around
|
||||
* {@link #convertPdfToJson(MultipartFile, OutputStream)} for callers that need to inspect or
|
||||
* mutate the document in memory before round-tripping it back to PDF.
|
||||
*/
|
||||
public PdfJsonDocument convertPdfToJsonDocument(MultipartFile file) throws IOException {
|
||||
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
|
||||
convertPdfToJson(file, null, false, buffer);
|
||||
return objectMapper.readValue(buffer.toByteArray(), PdfJsonDocument.class);
|
||||
}
|
||||
|
||||
public void convertPdfToJson(MultipartFile file, boolean lightweight, OutputStream out)
|
||||
throws IOException {
|
||||
convertPdfToJson(file, null, lightweight, out);
|
||||
@@ -620,6 +631,13 @@ public class PdfJsonConversionService {
|
||||
}
|
||||
byte[] jsonBytes = file.getBytes();
|
||||
PdfJsonDocument pdfJson = objectMapper.readValue(jsonBytes, PdfJsonDocument.class);
|
||||
convertJsonToPdf(pdfJson, out);
|
||||
}
|
||||
|
||||
public void convertJsonToPdf(PdfJsonDocument pdfJson, OutputStream out) throws IOException {
|
||||
if (pdfJson == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("document");
|
||||
}
|
||||
|
||||
List<PdfJsonFont> fontModels = pdfJson.getFonts();
|
||||
if (fontModels == null) {
|
||||
@@ -2076,14 +2094,29 @@ public class PdfJsonConversionService {
|
||||
byte[] toUnicodeBytes = Base64.getDecoder().decode(toUnicodeBase64);
|
||||
String toUnicodeStr = new String(toUnicodeBytes, StandardCharsets.UTF_8);
|
||||
|
||||
// Parse ToUnicode CMap for bfchar and bfrange
|
||||
// Parse ToUnicode CMap for bfchar and bfrange. Both sides go through
|
||||
// parseToUnicodeCodepoint so an 8-hex-char surrogate-pair value on either side
|
||||
// (e.g. "D837DF0E") is decoded as a single supplementary codepoint instead of
|
||||
// overflowing Integer.parseInt and aborting the whole mapping build.
|
||||
java.util.regex.Pattern bfcharPattern =
|
||||
java.util.regex.Pattern.compile("<([0-9A-Fa-f]+)>\\s*<([0-9A-Fa-f]+)>");
|
||||
java.util.regex.Matcher matcher = bfcharPattern.matcher(toUnicodeStr);
|
||||
while (matcher.find()) {
|
||||
int charCode = Integer.parseInt(matcher.group(1), 16);
|
||||
int unicode = Integer.parseInt(matcher.group(2), 16);
|
||||
charCodeToUnicode.put(charCode, unicode);
|
||||
try {
|
||||
int charCode = parseToUnicodeCodepoint(matcher.group(1));
|
||||
int unicode = parseToUnicodeCodepoint(matcher.group(2));
|
||||
charCodeToUnicode.put(charCode, unicode);
|
||||
} catch (NumberFormatException entryEx) {
|
||||
// Tolerate a single malformed entry: log and skip rather than aborting the
|
||||
// entire ToUnicode CMap (which would force the whole font onto the slow
|
||||
// raw-bytes fallback path).
|
||||
log.debug(
|
||||
"Skipping malformed ToUnicode entry <{}> <{}> in font {}: {}",
|
||||
matcher.group(1),
|
||||
matcher.group(2),
|
||||
font.getName(),
|
||||
entryEx.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Build JSON mapping: CharCode → CID → GID → Unicode
|
||||
@@ -2136,6 +2169,42 @@ public class PdfJsonConversionService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a hex string from a PDF ToUnicode CMap into a single Unicode codepoint. Handles three
|
||||
* cases: a single BMP code unit (4 hex chars), a UTF-16 surrogate pair encoding a supplementary
|
||||
* codepoint above U+FFFF (8 hex chars, e.g. {@code D837DF0E} for U+1F40E), and multi-codepoint
|
||||
* mappings (longer; returns the first codepoint as a best-effort representative).
|
||||
*
|
||||
* <p>Without this, {@code Integer.parseInt("D837DF0E", 16)} overflows because the value is ~3.6
|
||||
* billion, throwing {@link NumberFormatException} and forcing the conversion to fall back to a
|
||||
* raw ToUnicode payload that the JSON→PDF rebuild then fails to use efficiently.
|
||||
*/
|
||||
static int parseToUnicodeCodepoint(String hex) {
|
||||
if (hex == null || hex.isEmpty()) {
|
||||
throw new NumberFormatException("Empty ToUnicode hex value");
|
||||
}
|
||||
if (hex.length() <= 4) {
|
||||
return Integer.parseInt(hex, 16);
|
||||
}
|
||||
// Treat the hex string as UTF-16BE: pairs of hex digits form bytes, four hex digits form
|
||||
// one UTF-16 code unit. The PDF ToUnicode CMap convention requires an even number of bytes
|
||||
// (i.e. a multiple of four hex characters) for multi-unit values.
|
||||
if (hex.length() % 4 != 0) {
|
||||
throw new NumberFormatException(
|
||||
"ToUnicode hex value not a multiple of 4 chars: " + hex);
|
||||
}
|
||||
int unitCount = hex.length() / 4;
|
||||
char[] units = new char[unitCount];
|
||||
for (int i = 0; i < unitCount; i++) {
|
||||
units[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16);
|
||||
}
|
||||
// codePointAt assembles a surrogate pair into a supplementary codepoint when the
|
||||
// high/low surrogates appear in sequence; for any other multi-unit sequence it returns
|
||||
// the first BMP codepoint, which is the right best-effort fallback for ligature
|
||||
// decompositions (one charCode -> several Unicode chars).
|
||||
return new String(units).codePointAt(0);
|
||||
}
|
||||
|
||||
private PdfJsonFontCidSystemInfo extractCidSystemInfo(COSDictionary fontDictionary) {
|
||||
if (fontDictionary == null) {
|
||||
return null;
|
||||
@@ -4256,12 +4325,8 @@ public class PdfJsonConversionService {
|
||||
return 0;
|
||||
}
|
||||
if (font != null) {
|
||||
try (InputStream inputStream = new ByteArrayInputStream(value.getBytes())) {
|
||||
int count = 0;
|
||||
int code;
|
||||
while ((code = font.readCode(inputStream)) != -1) {
|
||||
count++;
|
||||
}
|
||||
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes())) {
|
||||
int count = countCodesProtected(inputStream, font::readCode);
|
||||
if (count > 0) {
|
||||
return count;
|
||||
}
|
||||
@@ -4273,6 +4338,49 @@ public class PdfJsonConversionService {
|
||||
return Math.max(1, bytes.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Functional accessor for {@link PDFont#readCode(InputStream)} so the bounded counting loop can
|
||||
* be exercised in isolation without instantiating a {@link PDFont}.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
interface CodeReader {
|
||||
int readCode(InputStream stream) throws IOException;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count how many codes the supplied {@code reader} can extract from {@code inputStream}, with
|
||||
* two safety nets that PDFBox's raw {@link PDFont#readCode(InputStream)} loop lacks:
|
||||
*
|
||||
* <ol>
|
||||
* <li>Stop when the stream is empty (a corrupt CMap can otherwise loop forever returning
|
||||
* successfully-matched zero-bytes from an exhausted {@link ByteArrayInputStream}).
|
||||
* <li>Stop when a {@code readCode} call did not consume any bytes, even if it returned a
|
||||
* non-{@code -1} value.
|
||||
* </ol>
|
||||
*
|
||||
* <p>Both conditions were observed in the wild on round-tripped fallback fonts where the
|
||||
* embedded ToUnicode CMap matched 0x00 sequences, hanging the JSON→PDF rebuild.
|
||||
*/
|
||||
static int countCodesProtected(ByteArrayInputStream inputStream, CodeReader reader)
|
||||
throws IOException {
|
||||
int count = 0;
|
||||
int previousAvailable = inputStream.available();
|
||||
while (previousAvailable > 0) {
|
||||
int code = reader.readCode(inputStream);
|
||||
if (code == -1) {
|
||||
break;
|
||||
}
|
||||
int currentAvailable = inputStream.available();
|
||||
if (currentAvailable >= previousAvailable) {
|
||||
// No progress made; break to avoid infinite loop on corrupt CMaps.
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
previousAvailable = currentAvailable;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private MergedText mergeText(List<PdfJsonTextElement> elements) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
List<Integer> combinedCodes = new ArrayList<>();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+664
@@ -0,0 +1,664 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.general.EditTextRequest;
|
||||
import stirling.software.SPDF.model.json.PdfJsonDocument;
|
||||
import stirling.software.SPDF.model.json.PdfJsonPage;
|
||||
import stirling.software.SPDF.model.json.PdfJsonTextElement;
|
||||
import stirling.software.SPDF.service.PdfJsonConversionService;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class EditTextControllerTest {
|
||||
|
||||
@Mock private PdfJsonConversionService pdfJsonConversionService;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
|
||||
@InjectMocks private EditTextController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile("test", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
}
|
||||
|
||||
private static MultipartFile pdfFile() {
|
||||
return new MockMultipartFile(
|
||||
"fileInput", "doc.pdf", "application/pdf", "stub-pdf-bytes".getBytes());
|
||||
}
|
||||
|
||||
private static EditTextOperation edit(String find, String replace) {
|
||||
EditTextOperation op = new EditTextOperation();
|
||||
op.setFind(find);
|
||||
op.setReplace(replace);
|
||||
return op;
|
||||
}
|
||||
|
||||
private static PdfJsonTextElement textElement(String text) {
|
||||
PdfJsonTextElement el = new PdfJsonTextElement();
|
||||
el.setText(text);
|
||||
el.setCharCodes(new int[] {1, 2, 3});
|
||||
return el;
|
||||
}
|
||||
|
||||
private static PdfJsonDocument documentWithElements(List<PdfJsonTextElement> elements) {
|
||||
PdfJsonDocument doc = new PdfJsonDocument();
|
||||
PdfJsonPage page = new PdfJsonPage();
|
||||
page.setPageNumber(1);
|
||||
page.setTextElements(new ArrayList<>(elements));
|
||||
doc.setPages(new ArrayList<>(List.of(page)));
|
||||
return doc;
|
||||
}
|
||||
|
||||
private static PdfJsonDocument documentWithText(String... textsByPage) {
|
||||
PdfJsonDocument doc = new PdfJsonDocument();
|
||||
List<PdfJsonPage> pages = new ArrayList<>();
|
||||
for (int i = 0; i < textsByPage.length; i++) {
|
||||
PdfJsonPage page = new PdfJsonPage();
|
||||
page.setPageNumber(i + 1);
|
||||
PdfJsonTextElement el = new PdfJsonTextElement();
|
||||
el.setText(textsByPage[i]);
|
||||
el.setCharCodes(new int[] {1, 2, 3});
|
||||
page.setTextElements(new ArrayList<>(List.of(el)));
|
||||
pages.add(page);
|
||||
}
|
||||
doc.setPages(pages);
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Build a minimal real PDF so tests can run end-to-end without mocking the conversion. */
|
||||
private static byte[] buildEmptyPdf() throws Exception {
|
||||
try (PDDocument doc = new PDDocument();
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
doc.addPage(new PDPage());
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_nullFileInputThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(null);
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_emptyEditsThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of());
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_nullEditsThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(null);
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_emptyFindStringThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("", "replacement")));
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_findStringWithRegexMetacharsIsTreatedLiterally() throws Exception {
|
||||
// Find strings are always treated as literals (Pattern.quote'd internally) — no ReDoS
|
||||
// exposure from regex metacharacters supplied by the caller.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("(unclosed", "fixed")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("text with (unclosed paren");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"text with fixed paren",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_literalFindReplace_mutatesMatchingSpansAndClearsCharCodes() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("foo and foo", "no match here");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
assertNotNull(response.getBody());
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
|
||||
assertEquals("bar and bar", mutated.getPages().get(0).getTextElements().get(0).getText());
|
||||
assertNull(mutated.getPages().get(0).getTextElements().get(0).getCharCodes());
|
||||
|
||||
assertEquals("no match here", mutated.getPages().get(1).getTextElements().get(0).getText());
|
||||
// Char codes preserved on unmodified spans.
|
||||
assertNotNull(mutated.getPages().get(1).getTextElements().get(0).getCharCodes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_wholeWordSearch_doesNotMatchInsideWords() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("cat", "dog")));
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
PdfJsonDocument input = documentWithText("the cat is in the catalogue");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"the dog is in the catalogue",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_wholeWordSearch_matchesFindStartingWithNonWordChar() throws Exception {
|
||||
// Regression: \b only fires on a word/non-word transition. A find that starts with a
|
||||
// non-word char (e.g. "-foo") preceded by another non-word char in the source (a space)
|
||||
// would never match under \b. The lookaround-based bound handles this correctly.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("-foo", "-bar")));
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
PdfJsonDocument input = documentWithText("space then -foo here");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"space then -bar here",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_wholeWordSearch_doesNotMatchWhenAdjacentToWordChar() throws Exception {
|
||||
// The lookaround bound must still reject matches that are part of a larger word.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("-foo", "-bar")));
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
PdfJsonDocument input = documentWithText("inline-foo should not match");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
// "inline-foo" has 'e' (word) before '-foo' so the lookbehind blocks the match. The
|
||||
// trailing 'o' is followed by a space (non-word) so the trailing lookahead would pass on
|
||||
// its own; the leading lookbehind is what rejects it.
|
||||
assertEquals(
|
||||
"inline-foo should not match",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_pageFilter_onlyAffectsListedPages() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
request.setPageNumbers("2");
|
||||
|
||||
PdfJsonDocument input = documentWithText("foo on page 1", "foo on page 2", "foo on page 3");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
|
||||
assertEquals("foo on page 1", mutated.getPages().get(0).getTextElements().get(0).getText());
|
||||
assertEquals("bar on page 2", mutated.getPages().get(1).getTextElements().get(0).getText());
|
||||
assertEquals("foo on page 3", mutated.getPages().get(2).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_pageRange_appliesToAllPagesInRange() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
request.setPageNumbers("2-3");
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithText("foo page 1", "foo page 2", "foo page 3", "foo page 4");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
|
||||
assertTrue(mutated.getPages().get(0).getTextElements().get(0).getText().startsWith("foo"));
|
||||
assertTrue(mutated.getPages().get(1).getTextElements().get(0).getText().startsWith("bar"));
|
||||
assertTrue(mutated.getPages().get(2).getTextElements().get(0).getText().startsWith("bar"));
|
||||
assertTrue(mutated.getPages().get(3).getTextElements().get(0).getText().startsWith("foo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_orderedEdits_applyInSequence() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar"), edit("bar", "baz")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("foo");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals("baz", captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_replaceWithEmptyString_deletesMatch() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("DRAFT ", "")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("DRAFT Confidential Memo");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"Confidential Memo",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_noMatches_returnsPdfWithoutMutation() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("notfound", "replacement")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("nothing to match");
|
||||
int[] originalCodes = input.getPages().get(0).getTextElements().get(0).getCharCodes();
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
// Char codes left intact when nothing was replaced.
|
||||
assertEquals(
|
||||
originalCodes, input.getPages().get(0).getTextElements().get(0).getCharCodes());
|
||||
assertEquals(
|
||||
"nothing to match", input.getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_dollarInLiteralReplacement_isQuoted() throws Exception {
|
||||
// Without quoting, '$1' would be interpreted as a backreference and crash.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("price", "$100")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("the price is set");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"the $100 is set",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_emptyDocument_returnsResponseWithoutErrors() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
PdfJsonDocument input = new PdfJsonDocument();
|
||||
input.setPages(new ArrayList<>());
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
assertNotNull(response.getBody());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_textElementWithNullText_isSkipped() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
PdfJsonDocument input = new PdfJsonDocument();
|
||||
PdfJsonPage page = new PdfJsonPage();
|
||||
page.setPageNumber(1);
|
||||
PdfJsonTextElement nullText = new PdfJsonTextElement();
|
||||
nullText.setText(null);
|
||||
PdfJsonTextElement realText = new PdfJsonTextElement();
|
||||
realText.setText("foo here");
|
||||
page.setTextElements(new ArrayList<>(List.of(nullText, realText)));
|
||||
input.setPages(new ArrayList<>(List.of(page)));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
assertNull(mutated.getPages().get(0).getTextElements().get(0).getText());
|
||||
assertEquals("bar here", mutated.getPages().get(0).getTextElements().get(1).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_matchSpansTwoElements() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(List.of(textElement("Hello "), textElement("World")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Whole replacement lands in the first matched element; the second is emptied. The
|
||||
// JSON->PDF rebuild concatenates per-token text so the font lays out the replacement as
|
||||
// one run anchored at the first element's X position.
|
||||
assertEquals("Goodbye Earth", elements.get(0).getText());
|
||||
assertEquals("", elements.get(1).getText());
|
||||
assertNull(elements.get(0).getCharCodes());
|
||||
assertNull(elements.get(1).getCharCodes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_matchSpansFiveElementsLikeFragmentedTitle() throws Exception {
|
||||
// Reproduces the real-world case: a multi-word title fragmented one word per text span.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(
|
||||
List.of(edit("The Free Adobe Acrobat Alternative", "The PDF automation pipeline")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("The "),
|
||||
textElement("Free "),
|
||||
textElement("Adobe "),
|
||||
textElement("Acrobat "),
|
||||
textElement("Alternative")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Whole replacement is written into the first matched element; the remaining four are
|
||||
// emptied. Centered titles will become left-aligned at the original first-word X position.
|
||||
assertEquals("The PDF automation pipeline", elements.get(0).getText());
|
||||
for (int i = 1; i < elements.size(); i++) {
|
||||
assertEquals("", elements.get(i).getText(), "element " + i + " should be empty");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_preservesPrefixAndSuffix() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(textElement("Greeting: Hello "), textElement("World! And more")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// First element keeps its prefix and gets the entire replacement; last element keeps its
|
||||
// suffix only.
|
||||
assertEquals("Greeting: Goodbye Earth", elements.get(0).getText());
|
||||
assertEquals("! And more", elements.get(1).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_matchInOneElementOfMany_onlyTouchesThatElement() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("World", "Earth")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("Hello "),
|
||||
textElement("World!"),
|
||||
textElement(" Goodbye")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
assertEquals("Hello ", elements.get(0).getText());
|
||||
assertEquals("Earth!", elements.get(1).getText());
|
||||
assertEquals(" Goodbye", elements.get(2).getText());
|
||||
// Only the modified element's char codes get cleared.
|
||||
assertNotNull(elements.get(0).getCharCodes());
|
||||
assertNull(elements.get(1).getCharCodes());
|
||||
assertNotNull(elements.get(2).getCharCodes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_multipleMatchesAppliedRightToLeft() throws Exception {
|
||||
// Two matches in the same page text. Right-to-left application keeps earlier indices
|
||||
// valid as later matches are written.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo bar", "X")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("foo "),
|
||||
textElement("bar baz "),
|
||||
textElement("foo "),
|
||||
textElement("bar")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Joined was "foo bar baz foo bar"; both "foo bar" runs replaced with "X".
|
||||
StringBuilder joined = new StringBuilder();
|
||||
for (PdfJsonTextElement el : elements) {
|
||||
joined.append(el.getText() == null ? "" : el.getText());
|
||||
}
|
||||
assertEquals("X baz X", joined.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_subWordFragmentation_writesIntoFirstElement() throws Exception {
|
||||
// When the matched text is split into many character-level spans (typical of Type3 glyph
|
||||
// runs), the whole replacement is dumped into the first matched element and the rest are
|
||||
// emptied. The font lays out the replacement as one run at the first glyph's X position.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
|
||||
|
||||
// 11 sub-word elements covering "Hello World".
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("H"),
|
||||
textElement("e"),
|
||||
textElement("l"),
|
||||
textElement("l"),
|
||||
textElement("o"),
|
||||
textElement(" "),
|
||||
textElement("W"),
|
||||
textElement("o"),
|
||||
textElement("r"),
|
||||
textElement("l"),
|
||||
textElement("d")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Entire replacement goes into the first matched element; the other 10 are emptied.
|
||||
assertEquals("Goodbye Earth", elements.get(0).getText());
|
||||
for (int i = 1; i < elements.size(); i++) {
|
||||
assertEquals("", elements.get(i).getText(), "element " + i + " should be empty");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_outputFilenameDerivedFromInput() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(
|
||||
new MockMultipartFile(
|
||||
"fileInput", "report.pdf", "application/pdf", buildEmptyPdf()));
|
||||
request.setEdits(List.of(edit("anything", "x")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("nothing matches");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
String contentDisposition =
|
||||
response.getHeaders()
|
||||
.getFirst(org.springframework.http.HttpHeaders.CONTENT_DISPOSITION);
|
||||
assertNotNull(contentDisposition);
|
||||
assertTrue(contentDisposition.contains("report_edited.pdf"));
|
||||
assertFalse(contentDisposition.contains(".pdf_edited.pdf"));
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -177,14 +177,14 @@ class GlobalExceptionHandlerTest {
|
||||
void handleMaxUploadSize_returns_413() {
|
||||
MaxUploadSizeExceededException ex = new MaxUploadSizeExceededException(10485760);
|
||||
ResponseEntity<ProblemDetail> resp = handler.handleMaxUploadSize(ex, request);
|
||||
assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, resp.getStatusCode());
|
||||
assertEquals(HttpStatus.CONTENT_TOO_LARGE, resp.getStatusCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void handleMaxUploadSize_unknown_limit() {
|
||||
MaxUploadSizeExceededException ex = new MaxUploadSizeExceededException(-1);
|
||||
ResponseEntity<ProblemDetail> resp = handler.handleMaxUploadSize(ex, request);
|
||||
assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, resp.getStatusCode());
|
||||
assertEquals(HttpStatus.CONTENT_TOO_LARGE, resp.getStatusCode());
|
||||
assertNull(resp.getBody().getProperties().get("maxSizeBytes"));
|
||||
}
|
||||
|
||||
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
package stirling.software.SPDF.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link PdfJsonConversionService#parseToUnicodeCodepoint(String)}.
|
||||
*
|
||||
* <p>The function exists because PDF ToUnicode CMap entries can encode supplementary-plane
|
||||
* codepoints (above U+FFFF) as UTF-16 surrogate pairs, e.g. {@code <D837DF0E>} for U+1F40E. The
|
||||
* naive implementation that called {@link Integer#parseInt(String, int)} on the whole hex string
|
||||
* threw {@link NumberFormatException} for any 8-char value because they overflow {@code int}. That
|
||||
* triggered a fallback that left fonts with no proper CID mapping and made downstream JSON→PDF
|
||||
* rebuilds slow/hung when rendering text with those fonts.
|
||||
*/
|
||||
class PdfJsonConversionServiceUnicodeParsingTest {
|
||||
|
||||
@Test
|
||||
void parsesSingleBmpCodeUnit() {
|
||||
// Latin capital A.
|
||||
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("0041"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesShortBmpHexValue() {
|
||||
// Some ToUnicode entries omit leading zeros for low codepoints.
|
||||
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("41"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesSupplementaryCodepointFromSurrogatePair() {
|
||||
// U+1F40E HORSE encoded as UTF-16 surrogate pair D83D DC0E. The bug we are fixing was
|
||||
// that Integer.parseInt("D83DDC0E", 16) overflows because D83DDC0E > Integer.MAX_VALUE.
|
||||
assertEquals(0x1F40E, PdfJsonConversionService.parseToUnicodeCodepoint("D83DDC0E"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesSurrogatePairFromUserReportedHang() {
|
||||
// The exact hex value from the user's hang reproducer log:
|
||||
// "Failed to build Unicode mapping for font NotoSans-Regular: For input string:
|
||||
// \"D837DF0E\" under radix 16"
|
||||
// D837 DF0E decodes to U+1DF0E (CJK supplementary). Important assertion: it returns a
|
||||
// valid codepoint instead of overflowing Integer.parseInt and throwing.
|
||||
int expected = new String(new char[] {(char) 0xD837, (char) 0xDF0E}).codePointAt(0);
|
||||
assertEquals(0x1DF0E, expected); // sanity check on the test setup itself
|
||||
assertEquals(expected, PdfJsonConversionService.parseToUnicodeCodepoint("D837DF0E"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesLigatureDecompositionAsFirstCodepoint() {
|
||||
// A ToUnicode entry can map one charCode to multiple Unicode chars (a ligature). PDF
|
||||
// spec allows e.g. <0041 0042> for "AB". Our best-effort behavior is to return the
|
||||
// first codepoint so the mapping is at least roughly meaningful for search/copy.
|
||||
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("00410042"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsEmptyHex() {
|
||||
assertThrows(
|
||||
NumberFormatException.class,
|
||||
() -> PdfJsonConversionService.parseToUnicodeCodepoint(""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsNullHex() {
|
||||
assertThrows(
|
||||
NumberFormatException.class,
|
||||
() -> PdfJsonConversionService.parseToUnicodeCodepoint(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsOddByteCountAboveBmp() {
|
||||
// 6 hex chars is 3 bytes — not a valid UTF-16BE sequence.
|
||||
assertThrows(
|
||||
NumberFormatException.class,
|
||||
() -> PdfJsonConversionService.parseToUnicodeCodepoint("D83DDC"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedTerminatesWhenReaderMakesNoProgress() {
|
||||
// Reproduces the user's hang: PDFBox's CMap.readCode can return a successful code (0)
|
||||
// from a stream where no bytes were consumed (corrupt codespace matching 0x00 bytes from
|
||||
// the buffer's uninitialized region after EOF). Without the no-progress guard, the
|
||||
// counting loop in countGlyphs ran forever.
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3, 4});
|
||||
PdfJsonConversionService.CodeReader reader = in -> 0; // never reads, always "succeeds"
|
||||
|
||||
int count =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(2),
|
||||
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
|
||||
|
||||
// First iteration sees no progress and breaks immediately.
|
||||
assertEquals(0, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedTerminatesOnEmptyStream() {
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]);
|
||||
PdfJsonConversionService.CodeReader reader =
|
||||
in -> {
|
||||
throw new AssertionError("reader must not be called when stream is empty");
|
||||
};
|
||||
|
||||
int count =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(2),
|
||||
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
|
||||
|
||||
assertEquals(0, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedHonorsExplicitMinusOneReturn() throws IOException {
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3});
|
||||
PdfJsonConversionService.CodeReader reader =
|
||||
in -> {
|
||||
int b = in.read();
|
||||
return b == -1 ? -1 : b;
|
||||
};
|
||||
|
||||
int count = PdfJsonConversionService.countCodesProtected(stream, reader);
|
||||
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedTerminatesIfReaderReadsThenStops() throws IOException {
|
||||
// A reader that consumes one byte then hits a corrupt-CMap pattern returning 0 without
|
||||
// consuming further must still terminate after counting the consumed bytes.
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3, 4});
|
||||
PdfJsonConversionService.CodeReader reader =
|
||||
new PdfJsonConversionService.CodeReader() {
|
||||
boolean firstCall = true;
|
||||
|
||||
@Override
|
||||
public int readCode(InputStream in) throws IOException {
|
||||
if (firstCall) {
|
||||
firstCall = false;
|
||||
return in.read();
|
||||
}
|
||||
return 0; // simulates corrupt CMap thereafter
|
||||
}
|
||||
};
|
||||
|
||||
int count =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(2),
|
||||
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
|
||||
|
||||
assertEquals(1, count);
|
||||
}
|
||||
}
|
||||
+38
-4
@@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.common.service.JobOwnershipService;
|
||||
import stirling.software.common.service.TaskManager;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowProgressEvent;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowResponse;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowResultFile;
|
||||
@@ -140,13 +141,32 @@ public class AiEngineController {
|
||||
}
|
||||
|
||||
private void runOrchestrationStream(AiWorkflowRequest request, SseEmitter emitter) {
|
||||
AiWorkflowService.ProgressListener listener =
|
||||
new AiWorkflowService.ProgressListener() {
|
||||
@Override
|
||||
public void onProgress(AiWorkflowProgressEvent event) {
|
||||
sendEvent(emitter, "progress", event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onHeartbeat() {
|
||||
// Forward upstream heartbeats so the SSE pipe stays visibly alive between
|
||||
// real progress events; if the frontend has gone away, sendEvent throws,
|
||||
// which propagates up through the stream consumer and closes our upstream
|
||||
// engine connection so the engine can cancel its in-flight workflow.
|
||||
sendEvent(emitter, "heartbeat", Map.of());
|
||||
}
|
||||
};
|
||||
try {
|
||||
AiWorkflowResponse result =
|
||||
aiWorkflowService.orchestrate(
|
||||
request, progress -> sendEvent(emitter, "progress", progress));
|
||||
AiWorkflowResponse result = aiWorkflowService.orchestrate(request, listener);
|
||||
registerFileResultAsJob(result);
|
||||
sendEvent(emitter, "result", result);
|
||||
emitter.complete();
|
||||
} catch (ClientDisconnectedException e) {
|
||||
// The frontend gave up mid-stream. The exception unwinding through orchestrate()
|
||||
// already closed the upstream engine connection (engine sees disconnect and cancels).
|
||||
// The emitter is already toast; nothing useful left to send.
|
||||
log.debug("Client disconnected mid-stream; aborting workflow", e);
|
||||
} catch (Exception e) {
|
||||
log.error("AI orchestration stream failed", e);
|
||||
// Emit an error frame for the frontend and then complete normally. Using
|
||||
@@ -192,7 +212,21 @@ public class AiEngineController {
|
||||
try {
|
||||
emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON));
|
||||
} catch (IOException e) {
|
||||
log.debug("Failed to send SSE event (client may have disconnected)", e);
|
||||
// Surface the disconnect so the streaming pipeline unwinds: callers higher up close
|
||||
// the upstream engine connection, which lets the engine cancel its in-flight workflow.
|
||||
// Without this, the engine would keep producing (and billing for) tokens whose results
|
||||
// nobody is reading.
|
||||
throw new ClientDisconnectedException("Client disconnected from SSE stream", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Thrown by {@link #sendEvent} when the SSE emitter's underlying connection is gone. Treated as
|
||||
* a signal to abort the workflow, not as an error to report.
|
||||
*/
|
||||
private static final class ClientDisconnectedException extends RuntimeException {
|
||||
ClientDisconnectedException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* Body for {@code POST /api/v1/documents} on the AI engine. Sent by Java when the engine reports
|
||||
* {@code need_ingest} and the requested document's extracted content must be stored before the
|
||||
* workflow can continue.
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AiDocumentIngestRequest {
|
||||
|
||||
private String documentId;
|
||||
|
||||
private String source;
|
||||
|
||||
private List<AiPageText> pageText;
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonSubTypes;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||
|
||||
/**
|
||||
* Typed engine-emitted progress detail, mirroring the Python {@code ProgressEvent} discriminated
|
||||
* union (see {@code engine/src/stirling/contracts/progress.py}). Carried inside {@link
|
||||
* AiWorkflowProgressEvent#getEngineDetail()} for {@link AiWorkflowPhase#ENGINE_PROGRESS} events.
|
||||
*
|
||||
* <p>Sealed so adding a new engine-side phase forces a matching subtype on the Java side instead of
|
||||
* silently passing through as an opaque map. The {@code phase} string is the discriminator and
|
||||
* stays on the wire so the frontend (which doesn't know about Java's class hierarchy) can switch on
|
||||
* it.
|
||||
*/
|
||||
@JsonTypeInfo(
|
||||
use = JsonTypeInfo.Id.NAME,
|
||||
include = JsonTypeInfo.As.EXISTING_PROPERTY,
|
||||
property = "phase",
|
||||
visible = true)
|
||||
@JsonSubTypes({
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocReadStarted.class,
|
||||
name = "whole_doc_read_started"),
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocSliceDone.class,
|
||||
name = "whole_doc_slice_done"),
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocCompressionRound.class,
|
||||
name = "whole_doc_compression_round"),
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocReadDone.class,
|
||||
name = "whole_doc_read_done"),
|
||||
})
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public sealed interface AiEngineProgressDetail {
|
||||
|
||||
String phase();
|
||||
|
||||
record WholeDocReadStarted(String phase, String question, int pages, int slices)
|
||||
implements AiEngineProgressDetail {}
|
||||
|
||||
record WholeDocSliceDone(
|
||||
String phase,
|
||||
int completed,
|
||||
int total,
|
||||
String pages,
|
||||
int durationMs,
|
||||
int excerpts,
|
||||
int facts)
|
||||
implements AiEngineProgressDetail {}
|
||||
|
||||
record WholeDocCompressionRound(String phase, int roundNumber, int notesIn, int groups)
|
||||
implements AiEngineProgressDetail {}
|
||||
|
||||
record WholeDocReadDone(String phase, int completed, int slices, double durationSeconds)
|
||||
implements AiEngineProgressDetail {}
|
||||
}
|
||||
+2
-2
@@ -4,11 +4,11 @@ import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** A single page of extracted text for RAG ingest requests. */
|
||||
/** A single page of extracted text for document ingest requests. */
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AiRagPageText {
|
||||
public class AiPageText {
|
||||
|
||||
private int pageNumber;
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* Body for {@code POST /api/v1/rag/documents} on the AI engine. Sent by Java when the engine
|
||||
* reports {@code need_ingest} and the requested document's extracted content must be stored before
|
||||
* the workflow can continue.
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AiRagIngestRequest {
|
||||
|
||||
private String documentId;
|
||||
|
||||
private String source;
|
||||
|
||||
private List<AiRagPageText> pageText;
|
||||
}
|
||||
+7
-1
@@ -9,7 +9,13 @@ public enum AiWorkflowPhase {
|
||||
CALLING_ENGINE("calling_engine"),
|
||||
EXTRACTING_CONTENT("extracting_content"),
|
||||
EXECUTING_TOOL("executing_tool"),
|
||||
PROCESSING("processing");
|
||||
PROCESSING("processing"),
|
||||
/**
|
||||
* Generic engine-emitted progress event (e.g. chunked-reasoner slice progress). The original
|
||||
* engine event JSON is carried in {@link AiWorkflowProgressEvent#getEngineDetail()}, including
|
||||
* a specific {@code phase} string the frontend can switch on.
|
||||
*/
|
||||
ENGINE_PROGRESS("engine_progress");
|
||||
|
||||
private final String value;
|
||||
|
||||
|
||||
+22
-2
@@ -23,8 +23,17 @@ public class AiWorkflowProgressEvent {
|
||||
/** Total number of plan steps, for {@link AiWorkflowPhase#EXECUTING_TOOL} events. */
|
||||
private Integer stepCount;
|
||||
|
||||
/**
|
||||
* Engine-emitted event payload, for {@link AiWorkflowPhase#ENGINE_PROGRESS} events. The payload
|
||||
* is a typed subtype keyed on its {@code phase} string (e.g. {@code "whole_doc_slice_done"})
|
||||
* carrying phase-specific fields (slice index, page range, durations, etc.) that the frontend
|
||||
* can render as detailed progress.
|
||||
*/
|
||||
private AiEngineProgressDetail engineDetail;
|
||||
|
||||
public static AiWorkflowProgressEvent of(AiWorkflowPhase phase) {
|
||||
return new AiWorkflowProgressEvent(phase, System.currentTimeMillis(), null, null, null);
|
||||
return new AiWorkflowProgressEvent(
|
||||
phase, System.currentTimeMillis(), null, null, null, null);
|
||||
}
|
||||
|
||||
public static AiWorkflowProgressEvent executingTool(String tool, int stepIndex, int stepCount) {
|
||||
@@ -33,6 +42,17 @@ public class AiWorkflowProgressEvent {
|
||||
System.currentTimeMillis(),
|
||||
tool,
|
||||
stepIndex,
|
||||
stepCount);
|
||||
stepCount,
|
||||
null);
|
||||
}
|
||||
|
||||
public static AiWorkflowProgressEvent engineProgress(AiEngineProgressDetail detail) {
|
||||
return new AiWorkflowProgressEvent(
|
||||
AiWorkflowPhase.ENGINE_PROGRESS,
|
||||
System.currentTimeMillis(),
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
detail);
|
||||
}
|
||||
}
|
||||
|
||||
+66
@@ -7,6 +7,8 @@ import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.net.http.HttpTimeoutException;
|
||||
import java.time.Duration;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@@ -83,6 +85,70 @@ public class AiEngineClient {
|
||||
return response.body();
|
||||
}
|
||||
|
||||
/**
|
||||
* POST a JSON body and consume the response as a stream of NDJSON lines. Each line is passed to
|
||||
* {@code lineConsumer} in arrival order; the call returns when the engine closes the stream.
|
||||
*
|
||||
* <p>This is the right shape for long-running orchestrator calls that emit incremental
|
||||
* progress. The total HTTP timeout is the long-running timeout (typically 600s+), but in
|
||||
* practice line arrival keeps the connection logically alive: as long as the engine emits
|
||||
* events, the work is progressing. Genuine engine hangs still hit the total timeout.
|
||||
*/
|
||||
public void streamPost(String path, String jsonBody, Consumer<String> lineConsumer)
|
||||
throws IOException {
|
||||
ApplicationProperties.AiEngine config = applicationProperties.getAiEngine();
|
||||
if (!config.isEnabled()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "AI engine is not enabled");
|
||||
}
|
||||
|
||||
String url = config.getUrl().stripTrailing() + path;
|
||||
Duration timeout = Duration.ofSeconds(config.getLongRunningTimeoutSeconds());
|
||||
log.debug(
|
||||
"Proxying AI engine streaming request to {} (timeout {}s)",
|
||||
url,
|
||||
timeout.toSeconds());
|
||||
|
||||
HttpRequest request =
|
||||
HttpRequest.newBuilder()
|
||||
.uri(URI.create(url))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Accept", "application/x-ndjson")
|
||||
.timeout(timeout)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(jsonBody))
|
||||
.build();
|
||||
|
||||
HttpResponse<Stream<String>> response;
|
||||
try {
|
||||
response = httpClient.send(request, HttpResponse.BodyHandlers.ofLines());
|
||||
} catch (HttpTimeoutException e) {
|
||||
throw new ResponseStatusException(HttpStatus.GATEWAY_TIMEOUT, "AI engine timed out", e);
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "AI engine unreachable: " + e.getMessage(), e);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "AI engine request was interrupted");
|
||||
}
|
||||
|
||||
int status = response.statusCode();
|
||||
if (status >= 400) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.valueOf(status >= 500 ? 502 : status),
|
||||
"AI engine returned error: " + status);
|
||||
}
|
||||
|
||||
try (Stream<String> lines = response.body()) {
|
||||
lines.forEach(
|
||||
line -> {
|
||||
if (!line.isEmpty()) {
|
||||
lineConsumer.accept(line);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public String get(String path) throws IOException {
|
||||
ApplicationProperties.AiEngine config = applicationProperties.getAiEngine();
|
||||
if (!config.isEnabled()) {
|
||||
|
||||
+110
-17
@@ -30,15 +30,17 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.service.InternalApiTimeoutException;
|
||||
import stirling.software.common.service.ToolMetadataService;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.ZipExtractionUtils;
|
||||
import stirling.software.proprietary.model.api.ai.AiConversationMessage;
|
||||
import stirling.software.proprietary.model.api.ai.AiDocumentIngestRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiEngineProgressDetail;
|
||||
import stirling.software.proprietary.model.api.ai.AiFile;
|
||||
import stirling.software.proprietary.model.api.ai.AiRagIngestRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiRagPageText;
|
||||
import stirling.software.proprietary.model.api.ai.AiPageText;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowFileInput;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowOutcome;
|
||||
@@ -60,7 +62,7 @@ import tools.jackson.databind.ObjectMapper;
|
||||
@RequiredArgsConstructor
|
||||
public class AiWorkflowService {
|
||||
|
||||
private static final String RAG_DOCUMENTS_ENDPOINT = "/api/v1/rag/documents";
|
||||
private static final String DOCUMENTS_ENDPOINT = "/api/v1/documents";
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final AiEngineClient aiEngineClient;
|
||||
@@ -76,6 +78,14 @@ public class AiWorkflowService {
|
||||
@FunctionalInterface
|
||||
public interface ProgressListener {
|
||||
void onProgress(AiWorkflowProgressEvent event);
|
||||
|
||||
/**
|
||||
* Called when the engine emits a keep-alive heartbeat. Default is a no-op; consumers that
|
||||
* forward to a downstream connection (e.g. an SSE emitter) override this to push a
|
||||
* heartbeat through, so the next downstream-disconnect surfaces immediately rather than
|
||||
* waiting for the next real progress event.
|
||||
*/
|
||||
default void onHeartbeat() {}
|
||||
}
|
||||
|
||||
private static final ProgressListener NOOP_LISTENER = event -> {};
|
||||
@@ -143,7 +153,7 @@ public class AiWorkflowService {
|
||||
ProgressListener listener)
|
||||
throws IOException {
|
||||
listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.CALLING_ENGINE));
|
||||
AiWorkflowResponse response = invokeOrchestrator(request);
|
||||
AiWorkflowResponse response = invokeOrchestrator(request, listener);
|
||||
return switch (response.getOutcome()) {
|
||||
case NEED_CONTENT -> onNeedContent(response, filesById, request, listener);
|
||||
case NEED_INGEST -> onNeedIngest(response, filesById, request, listener);
|
||||
@@ -277,22 +287,22 @@ public class AiWorkflowService {
|
||||
}
|
||||
|
||||
private void ingestFile(AiFile file, MultipartFile multipartFile) throws IOException {
|
||||
List<AiRagPageText> pages = new ArrayList<>();
|
||||
List<AiPageText> pages = new ArrayList<>();
|
||||
try (PDDocument document = pdfDocumentFactory.load(multipartFile, true)) {
|
||||
int pageCount = document.getNumberOfPages();
|
||||
for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
|
||||
String pageText = pdfContentExtractor.extractPageTextRaw(document, pageNumber);
|
||||
if (pageText != null && !pageText.isBlank()) {
|
||||
pages.add(new AiRagPageText(pageNumber, pageText));
|
||||
pages.add(new AiPageText(pageNumber, pageText));
|
||||
}
|
||||
}
|
||||
}
|
||||
AiRagIngestRequest ingestRequest =
|
||||
new AiRagIngestRequest(file.getId(), file.getName(), pages);
|
||||
AiDocumentIngestRequest ingestRequest =
|
||||
new AiDocumentIngestRequest(file.getId(), file.getName(), pages);
|
||||
String body = objectMapper.writeValueAsString(ingestRequest);
|
||||
aiEngineClient.postLongRunning(RAG_DOCUMENTS_ENDPOINT, body);
|
||||
aiEngineClient.postLongRunning(DOCUMENTS_ENDPOINT, body);
|
||||
log.debug(
|
||||
"Ingested file into RAG: id={}, name={}, pages={}",
|
||||
"Ingested document: id={}, name={}, pages={}",
|
||||
file.getId(),
|
||||
file.getName(),
|
||||
pages.size());
|
||||
@@ -323,10 +333,12 @@ public class AiWorkflowService {
|
||||
result.files(),
|
||||
inputFileNames(filesById),
|
||||
result.report()));
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
log.error("Tool {} timed out: {}", endpointPath, e.getMessage());
|
||||
return new WorkflowState.Terminal(cannotContinue(toolTimeoutMessage(endpointPath, e)));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to execute tool {}: {}", endpointPath, e.getMessage(), e);
|
||||
return new WorkflowState.Terminal(
|
||||
cannotContinue("Tool execution failed: " + e.getMessage()));
|
||||
return new WorkflowState.Terminal(cannotContinue(toolFailureMessage(endpointPath, e)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +426,10 @@ public class AiWorkflowService {
|
||||
return new WorkflowState.Terminal(
|
||||
buildCompletedResponse(
|
||||
summary, currentFiles, inputFileNames(filesById), lastReport));
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage());
|
||||
return new WorkflowState.Terminal(
|
||||
cannotContinue(toolTimeoutMessage(e.getEndpointPath(), e)));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to execute plan: {}", e.getMessage(), e);
|
||||
return new WorkflowState.Terminal(
|
||||
@@ -425,6 +441,20 @@ public class AiWorkflowService {
|
||||
return filesById.values().stream().map(MultipartFile::getOriginalFilename).toList();
|
||||
}
|
||||
|
||||
private static String toolTimeoutMessage(String endpointPath, InternalApiTimeoutException e) {
|
||||
return String.format(
|
||||
"The %s tool did not respond within %d seconds and was aborted. The underlying"
|
||||
+ " operation may be hung; try again, run on a smaller file, or use a"
|
||||
+ " different approach.",
|
||||
endpointPath, e.getReadTimeout().toSeconds());
|
||||
}
|
||||
|
||||
private static String toolFailureMessage(String endpointPath, Throwable cause) {
|
||||
String reason =
|
||||
cause.getMessage() != null ? cause.getMessage() : cause.getClass().getSimpleName();
|
||||
return String.format("The %s tool failed: %s", endpointPath, reason);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single tool step. If the endpoint accepts multiple files, all files are sent in one
|
||||
* call. Otherwise, the endpoint is called once per file. ZIP responses are unpacked so each
|
||||
@@ -477,8 +507,15 @@ public class AiWorkflowService {
|
||||
}
|
||||
for (Map.Entry<String, Object> entry : parameters.entrySet()) {
|
||||
if (entry.getValue() instanceof List<?> list) {
|
||||
for (Object item : list) {
|
||||
body.add(entry.getKey(), item);
|
||||
if (containsStructuredElements(list)) {
|
||||
// Endpoints binding lists of structured objects (e.g. /security/redact's
|
||||
// redactions, /general/edit-text's edits) parse a single JSON string field via
|
||||
// a property editor. Pre-serialize the whole list so binding succeeds.
|
||||
body.add(entry.getKey(), objectMapper.writeValueAsString(list));
|
||||
} else {
|
||||
for (Object item : list) {
|
||||
body.add(entry.getKey(), item);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
body.add(entry.getKey(), entry.getValue());
|
||||
@@ -529,6 +566,15 @@ public class AiWorkflowService {
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean containsStructuredElements(List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Map<?, ?> || item instanceof List<?>) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private List<Resource> toResources(Map<String, MultipartFile> filesById) throws IOException {
|
||||
List<Resource> resources = new ArrayList<>();
|
||||
for (MultipartFile file : filesById.values()) {
|
||||
@@ -614,10 +660,57 @@ public class AiWorkflowService {
|
||||
return response;
|
||||
}
|
||||
|
||||
private AiWorkflowResponse invokeOrchestrator(WorkflowTurnRequest request) throws IOException {
|
||||
/**
|
||||
* Drive the engine's streaming orchestrator endpoint. Progress events are forwarded to {@code
|
||||
* listener} as they arrive (each one keeps the SSE connection to the frontend alive too). The
|
||||
* final {@code result} event carries the full {@link AiWorkflowResponse}; an {@code error}
|
||||
* event surfaces engine-side failures.
|
||||
*/
|
||||
private AiWorkflowResponse invokeOrchestrator(
|
||||
WorkflowTurnRequest request, ProgressListener listener) throws IOException {
|
||||
String requestBody = objectMapper.writeValueAsString(request);
|
||||
String responseBody = aiEngineClient.post("/api/v1/orchestrator", requestBody);
|
||||
return objectMapper.readValue(responseBody, AiWorkflowResponse.class);
|
||||
AiWorkflowResponse[] resultHolder = new AiWorkflowResponse[1];
|
||||
String[] errorHolder = new String[1];
|
||||
|
||||
aiEngineClient.streamPost(
|
||||
"/api/v1/orchestrator",
|
||||
requestBody,
|
||||
line -> handleStreamLine(line, listener, resultHolder, errorHolder));
|
||||
|
||||
if (errorHolder[0] != null) {
|
||||
throw new IOException("AI engine returned error: " + errorHolder[0]);
|
||||
}
|
||||
if (resultHolder[0] == null) {
|
||||
throw new IOException("AI engine stream ended without a result");
|
||||
}
|
||||
return resultHolder[0];
|
||||
}
|
||||
|
||||
private void handleStreamLine(
|
||||
String line,
|
||||
ProgressListener listener,
|
||||
AiWorkflowResponse[] resultHolder,
|
||||
String[] errorHolder) {
|
||||
try {
|
||||
JsonNode node = objectMapper.readTree(line);
|
||||
String event = node.path("event").asText();
|
||||
switch (event) {
|
||||
case "progress" -> {
|
||||
AiEngineProgressDetail detail =
|
||||
objectMapper.treeToValue(node, AiEngineProgressDetail.class);
|
||||
listener.onProgress(AiWorkflowProgressEvent.engineProgress(detail));
|
||||
}
|
||||
case "result" -> {
|
||||
JsonNode response = node.path("response");
|
||||
resultHolder[0] = objectMapper.treeToValue(response, AiWorkflowResponse.class);
|
||||
}
|
||||
case "error" -> errorHolder[0] = node.path("message").asText("unknown error");
|
||||
case "heartbeat" -> listener.onHeartbeat();
|
||||
default -> log.warn("Ignoring unknown engine stream event: {}", event);
|
||||
}
|
||||
} catch (JacksonException e) {
|
||||
log.warn("Failed to parse engine stream line: {}", line, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
|
||||
+203
-19
@@ -2,11 +2,13 @@ package stirling.software.proprietary.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
@@ -21,6 +23,7 @@ import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@@ -30,6 +33,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
@@ -44,6 +48,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.common.service.FileStorage.StoredFile;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.service.InternalApiTimeoutException;
|
||||
import stirling.software.common.service.ToolMetadataService;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.TempFileRegistry;
|
||||
@@ -251,6 +256,158 @@ class AiWorkflowServiceTest {
|
||||
verify(internalApiClient, times(1)).post(eq(COMPRESS_ENDPOINT), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void parameterListOfStructuredObjectsIsJsonEncodedAsSingleField() throws IOException {
|
||||
// Endpoints like /general/edit-text and /security/redact bind a List<StructuredObject>
|
||||
// from a single JSON form field via a property editor. The plan executor must
|
||||
// pre-serialize such lists rather than splitting them into repeated form fields.
|
||||
String editTextEndpoint = "/api/v1/general/edit-text";
|
||||
MockMultipartFile input = pdf("input.pdf", "bytes");
|
||||
stubOrchestrator(
|
||||
"""
|
||||
{
|
||||
"outcome":"plan",
|
||||
"summary":"Find and replace",
|
||||
"steps":[
|
||||
{"tool":"%s","parameters":{
|
||||
"edits":[{"find":"foo","replace":"bar"},{"find":"baz","replace":"qux"}],
|
||||
"useRegex":false
|
||||
}}
|
||||
]
|
||||
}
|
||||
"""
|
||||
.formatted(editTextEndpoint));
|
||||
when(toolMetadataService.isMultiInput(editTextEndpoint)).thenReturn(false);
|
||||
when(toolMetadataService.shouldUnpackZipResponse(editTextEndpoint)).thenReturn(false);
|
||||
stubEndpoint(editTextEndpoint, pdfResource("edited", "edited.pdf"));
|
||||
stubFileStorage();
|
||||
|
||||
service.orchestrate(requestFor(input, "find and replace"));
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<MultiValueMap<String, Object>> bodyCaptor =
|
||||
ArgumentCaptor.forClass(MultiValueMap.class);
|
||||
verify(internalApiClient).post(eq(editTextEndpoint), bodyCaptor.capture());
|
||||
MultiValueMap<String, Object> body = bodyCaptor.getValue();
|
||||
|
||||
// The structured-list field must be serialized as ONE JSON-string entry, not multiple
|
||||
// entries.
|
||||
List<Object> editsValues = body.get("edits");
|
||||
assertNotNull(editsValues);
|
||||
assertEquals(1, editsValues.size());
|
||||
assertEquals(
|
||||
"[{\"find\":\"foo\",\"replace\":\"bar\"},{\"find\":\"baz\",\"replace\":\"qux\"}]",
|
||||
editsValues.get(0));
|
||||
|
||||
// Primitive fields keep the original behavior (single value, not JSON-wrapped).
|
||||
List<Object> useRegex = body.get("useRegex");
|
||||
assertNotNull(useRegex);
|
||||
assertEquals(1, useRegex.size());
|
||||
assertEquals(false, useRegex.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parameterListOfPrimitivesIsSentAsRepeatedFormFields() throws IOException {
|
||||
// Endpoints like /misc/ocr-pdf bind List<String> via Spring's repeated-form-field
|
||||
// convention. The executor must preserve that behavior for primitive lists.
|
||||
String ocrEndpoint = "/api/v1/misc/ocr-pdf";
|
||||
MockMultipartFile input = pdf("input.pdf", "bytes");
|
||||
stubOrchestrator(
|
||||
"""
|
||||
{
|
||||
"outcome":"plan",
|
||||
"summary":"OCR the document",
|
||||
"steps":[
|
||||
{"tool":"%s","parameters":{"languages":["eng","fra","deu"]}}
|
||||
]
|
||||
}
|
||||
"""
|
||||
.formatted(ocrEndpoint));
|
||||
when(toolMetadataService.isMultiInput(ocrEndpoint)).thenReturn(false);
|
||||
when(toolMetadataService.shouldUnpackZipResponse(ocrEndpoint)).thenReturn(false);
|
||||
stubEndpoint(ocrEndpoint, pdfResource("ocred", "ocred.pdf"));
|
||||
stubFileStorage();
|
||||
|
||||
service.orchestrate(requestFor(input, "ocr"));
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<MultiValueMap<String, Object>> bodyCaptor =
|
||||
ArgumentCaptor.forClass(MultiValueMap.class);
|
||||
verify(internalApiClient).post(eq(ocrEndpoint), bodyCaptor.capture());
|
||||
List<Object> languages = bodyCaptor.getValue().get("languages");
|
||||
assertNotNull(languages);
|
||||
assertEquals(List.of("eng", "fra", "deu"), languages);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toolTimeoutSurfacesAsCannotContinueWithFriendlyMessage() throws IOException {
|
||||
// When an internal tool hangs, InternalApiClient throws InternalApiTimeoutException after
|
||||
// its read timeout fires. The workflow must convert that into a clean CANNOT_CONTINUE
|
||||
// outcome so the user sees an actionable message rather than the request hanging forever.
|
||||
MockMultipartFile input = pdf("input.pdf", "bytes");
|
||||
stubOrchestrator(
|
||||
"""
|
||||
{"outcome":"tool_call","tool":"%s","parameters":{"angle":90},"rationale":"Rotating"}
|
||||
"""
|
||||
.formatted(ROTATE_ENDPOINT));
|
||||
when(toolMetadataService.isMultiInput(ROTATE_ENDPOINT)).thenReturn(false);
|
||||
when(internalApiClient.post(eq(ROTATE_ENDPOINT), any()))
|
||||
.thenThrow(
|
||||
new InternalApiTimeoutException(
|
||||
ROTATE_ENDPOINT,
|
||||
java.time.Duration.ofSeconds(300),
|
||||
new java.io.IOException("Read timed out")));
|
||||
|
||||
AiWorkflowResponse result = service.orchestrate(requestFor(input, "rotate"));
|
||||
|
||||
assertEquals(AiWorkflowOutcome.CANNOT_CONTINUE, result.getOutcome());
|
||||
assertNotNull(result.getReason());
|
||||
assertTrue(
|
||||
result.getReason().contains(ROTATE_ENDPOINT),
|
||||
"reason should mention the failing tool path: " + result.getReason());
|
||||
assertTrue(
|
||||
result.getReason().contains("300"),
|
||||
"reason should mention the timeout duration: " + result.getReason());
|
||||
verify(internalApiClient, times(1)).post(eq(ROTATE_ENDPOINT), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void planTimeoutOnLaterStepStillSurfacesCleanly() throws IOException {
|
||||
// Multi-step plans must also handle a hung tool gracefully (no partial leaks) and the
|
||||
// failure message must identify which step failed so the user can iterate.
|
||||
MockMultipartFile input = pdf("input.pdf", "bytes");
|
||||
stubOrchestrator(
|
||||
"""
|
||||
{
|
||||
"outcome":"plan",
|
||||
"summary":"Rotate then compress",
|
||||
"steps":[
|
||||
{"tool":"%s","parameters":{"angle":90}},
|
||||
{"tool":"%s","parameters":{}}
|
||||
]
|
||||
}
|
||||
"""
|
||||
.formatted(ROTATE_ENDPOINT, COMPRESS_ENDPOINT));
|
||||
when(toolMetadataService.isMultiInput(anyString())).thenReturn(false);
|
||||
when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false);
|
||||
stubEndpoint(ROTATE_ENDPOINT, pdfResource("rotated", "rotated.pdf"));
|
||||
// No fileStorage stub here: the second step times out before any output reaches storage.
|
||||
when(internalApiClient.post(eq(COMPRESS_ENDPOINT), any()))
|
||||
.thenThrow(
|
||||
new InternalApiTimeoutException(
|
||||
COMPRESS_ENDPOINT,
|
||||
java.time.Duration.ofSeconds(300),
|
||||
new java.io.IOException("Read timed out")));
|
||||
|
||||
AiWorkflowResponse result = service.orchestrate(requestFor(input, "rotate then compress"));
|
||||
|
||||
assertEquals(AiWorkflowOutcome.CANNOT_CONTINUE, result.getOutcome());
|
||||
assertTrue(result.getReason().contains(COMPRESS_ENDPOINT));
|
||||
// The first step actually executed; the failure happened on the second.
|
||||
verify(internalApiClient, times(1)).post(eq(ROTATE_ENDPOINT), any());
|
||||
verify(internalApiClient, times(1)).post(eq(COMPRESS_ENDPOINT), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException {
|
||||
MockMultipartFile input = pdf("input.pdf", "bytes");
|
||||
@@ -264,7 +421,7 @@ class AiWorkflowServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void needIngestExtractsPageTextAndPostsToRagThenRetries() throws IOException {
|
||||
void needIngestExtractsPageTextAndPostsThenRetries() throws IOException {
|
||||
MockMultipartFile input = pdf("report.pdf", "bytes");
|
||||
when(fileIdStrategy.idFor(any())).thenReturn("report-id");
|
||||
|
||||
@@ -276,37 +433,64 @@ class AiWorkflowServiceTest {
|
||||
.thenReturn("page content");
|
||||
|
||||
int[] orchestratorCalls = {0};
|
||||
when(aiEngineClient.post(eq("/api/v1/orchestrator"), anyString()))
|
||||
.thenAnswer(
|
||||
doAnswer(
|
||||
inv -> {
|
||||
orchestratorCalls[0]++;
|
||||
String responseJson;
|
||||
if (orchestratorCalls[0] == 1) {
|
||||
return """
|
||||
{
|
||||
"outcome":"need_ingest",
|
||||
"resumeWith":"pdf_question",
|
||||
"reason":"ingest first",
|
||||
"filesToIngest":[{"id":"report-id","name":"report.pdf"}],
|
||||
"contentTypes":["page_text"]
|
||||
}
|
||||
""";
|
||||
responseJson =
|
||||
"""
|
||||
{
|
||||
"outcome":"need_ingest",
|
||||
"resumeWith":"pdf_question",
|
||||
"reason":"ingest first",
|
||||
"filesToIngest":[{"id":"report-id","name":"report.pdf"}],
|
||||
"contentTypes":["page_text"]
|
||||
}
|
||||
""";
|
||||
} else {
|
||||
responseJson =
|
||||
"""
|
||||
{"outcome":"answer","answer":"done","evidence":[]}
|
||||
""";
|
||||
}
|
||||
return """
|
||||
{"outcome":"answer","answer":"done","evidence":[]}
|
||||
""";
|
||||
});
|
||||
Consumer<String> consumer = inv.getArgument(2);
|
||||
consumer.accept(wrapAsResultEvent(responseJson));
|
||||
return null;
|
||||
})
|
||||
.when(aiEngineClient)
|
||||
.streamPost(eq("/api/v1/orchestrator"), anyString(), any());
|
||||
|
||||
AiWorkflowResponse result = service.orchestrate(requestFor(input, "summarise this"));
|
||||
|
||||
assertEquals(AiWorkflowOutcome.ANSWER, result.getOutcome());
|
||||
verify(aiEngineClient, times(1)).postLongRunning(eq("/api/v1/rag/documents"), anyString());
|
||||
verify(aiEngineClient, times(2)).post(eq("/api/v1/orchestrator"), anyString());
|
||||
verify(aiEngineClient, times(1)).postLongRunning(eq("/api/v1/documents"), anyString());
|
||||
verify(aiEngineClient, times(2)).streamPost(eq("/api/v1/orchestrator"), anyString(), any());
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
private void stubOrchestrator(String responseJson) throws IOException {
|
||||
when(aiEngineClient.post(eq("/api/v1/orchestrator"), anyString())).thenReturn(responseJson);
|
||||
doAnswer(
|
||||
inv -> {
|
||||
Consumer<String> consumer = inv.getArgument(2);
|
||||
consumer.accept(wrapAsResultEvent(responseJson));
|
||||
return null;
|
||||
})
|
||||
.when(aiEngineClient)
|
||||
.streamPost(eq("/api/v1/orchestrator"), anyString(), any());
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a unary orchestrator response JSON as the NDJSON "result" event the streaming endpoint
|
||||
* emits, so existing tests can keep their per-outcome JSON literals.
|
||||
*/
|
||||
private String wrapAsResultEvent(String responseJson) throws IOException {
|
||||
return objectMapper
|
||||
.createObjectNode()
|
||||
.put("event", "result")
|
||||
.set("response", objectMapper.readTree(responseJson))
|
||||
.toString();
|
||||
}
|
||||
|
||||
private void stubEndpoint(String endpoint, Resource body) {
|
||||
|
||||
+11
-8
@@ -20,10 +20,10 @@ import org.gradle.api.tasks.testing.Test
|
||||
import org.gradle.jvm.toolchain.JavaLanguageVersion
|
||||
|
||||
ext {
|
||||
springBootVersion = "4.0.5"
|
||||
springBootVersion = "4.0.6"
|
||||
pdfboxVersion = "3.0.7"
|
||||
imageioVersion = "3.13.1"
|
||||
lombokVersion = "1.18.44"
|
||||
lombokVersion = "1.18.46"
|
||||
bouncycastleVersion = "1.83"
|
||||
springSecuritySamlVersion = "7.0.5"
|
||||
openSamlVersion = "5.2.1"
|
||||
@@ -78,7 +78,7 @@ springBoot {
|
||||
|
||||
allprojects {
|
||||
group = 'stirling.software'
|
||||
version = '2.10.1'
|
||||
version = '2.11.0'
|
||||
|
||||
configurations.configureEach {
|
||||
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
|
||||
@@ -101,11 +101,14 @@ tasks.register('syncAppVersion') {
|
||||
|
||||
def tauriConfigFile = new File(tauriConfigPath)
|
||||
if (tauriConfigFile.exists()) {
|
||||
def parsed = new groovy.json.JsonSlurper().parse(tauriConfigFile)
|
||||
parsed.version = appVersionStr
|
||||
def formatted = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(parsed)) + System.lineSeparator()
|
||||
if (tauriConfigFile.getText('UTF-8') != formatted) {
|
||||
tauriConfigFile.write(formatted, 'UTF-8')
|
||||
def content = tauriConfigFile.getText('UTF-8')
|
||||
def matcher = (content =~ /(?m)^(\s*"version":\s*")([^"]*)(")/)
|
||||
if (!matcher.find()) {
|
||||
throw new GradleException("Could not locate version in ${tauriConfigFile} for synchronization")
|
||||
}
|
||||
def updatedContent = matcher.replaceFirst("${matcher.group(1)}${appVersionStr}${matcher.group(3)}")
|
||||
if (content != updatedContent) {
|
||||
tauriConfigFile.write(updatedContent, 'UTF-8')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -97,7 +97,7 @@ To add a 3rd UNO server:
|
||||
```yaml
|
||||
unoserver3:
|
||||
container_name: UNO-Server-3
|
||||
image: ghcr.io/unoconv/unoserver-docker:0.4.4
|
||||
image: ghcr.io/stirling-tools/stirling-unoserver:latest
|
||||
# ... same config as unoserver1/2
|
||||
```
|
||||
|
||||
|
||||
@@ -57,38 +57,34 @@ services:
|
||||
|
||||
unoserver1:
|
||||
container_name: UNO-Server-1
|
||||
# NOTE: This image needs to be updated to unoserver 3.6 to match Stirling-PDF's client version
|
||||
# Current :latest uses 3.4 which causes API mismatch errors
|
||||
image: ghcr.io/unoconv/unoserver-docker:latest
|
||||
# See docker/unoserver/VERSION for available tags.
|
||||
image: ghcr.io/stirling-tools/stirling-unoserver:latest
|
||||
environment:
|
||||
UNOSERVER_PORT: "2003"
|
||||
UNOSERVER_UNO_PORT: "2002"
|
||||
UNOSERVER_CONVERSION_TIMEOUT: "1800"
|
||||
# Recycle hourly to bound LibreOffice memory; 0 disables, floor 60s.
|
||||
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
|
||||
volumes:
|
||||
- stirling-tmp:/tmp/stirling-pdf:rw
|
||||
expose:
|
||||
- "2003"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/2003' || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
networks:
|
||||
- stirling-network
|
||||
restart: on-failure:5
|
||||
|
||||
unoserver2:
|
||||
container_name: UNO-Server-2
|
||||
# NOTE: This image needs to be updated to unoserver 3.6 to match Stirling-PDF's client version
|
||||
# Current :latest uses 3.4 which causes API mismatch errors
|
||||
image: ghcr.io/unoconv/unoserver-docker:latest
|
||||
image: ghcr.io/stirling-tools/stirling-unoserver:latest
|
||||
environment:
|
||||
UNOSERVER_PORT: "2003"
|
||||
UNOSERVER_UNO_PORT: "2002"
|
||||
UNOSERVER_CONVERSION_TIMEOUT: "1800"
|
||||
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
|
||||
volumes:
|
||||
- stirling-tmp:/tmp/stirling-pdf:rw
|
||||
expose:
|
||||
- "2003"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/2003' || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
networks:
|
||||
- stirling-network
|
||||
restart: on-failure:5
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# Standalone unoserver image for Stirling-PDF remote UNO mode.
|
||||
# Pinned to unoserver 3.6 to match Stirling-PDF's client (avoids 3.4/3.6 wire mismatch).
|
||||
|
||||
FROM ubuntu:noble@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b
|
||||
|
||||
ARG UNOSERVER_VERSION=3.6
|
||||
# ~120 MB of CJK fonts — opt-in.
|
||||
ARG INSTALL_CJK_FONTS=false
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
LANG=C.UTF-8 \
|
||||
LC_ALL=C.UTF-8 \
|
||||
UNOSERVER_PORT=2003 \
|
||||
UNOSERVER_UNO_PORT=2002 \
|
||||
UNOSERVER_INTERFACE=0.0.0.0 \
|
||||
UNOSERVER_CONVERSION_TIMEOUT=1800 \
|
||||
UNOSERVER_RECYCLE_INTERVAL_SECONDS=0 \
|
||||
UNOSERVER_PROFILE_DIR=/var/lib/unoserver/profile
|
||||
|
||||
RUN set -eux; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
software-properties-common ca-certificates curl gnupg tini; \
|
||||
add-apt-repository -y ppa:libreoffice/ppa; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 python3-venv python3-uno python3-packaging \
|
||||
libreoffice-writer-nogui libreoffice-calc-nogui \
|
||||
libreoffice-impress-nogui libreoffice-draw-nogui \
|
||||
libreoffice-base-nogui libreoffice-java-common \
|
||||
xvfb x11-utils coreutils \
|
||||
fonts-dejavu fonts-liberation fonts-liberation2 \
|
||||
fonts-crosextra-caladea fonts-crosextra-carlito \
|
||||
fonts-noto-core fonts-noto-mono fonts-noto-color-emoji \
|
||||
hyphen-en-gb hyphen-en-us hyphen-fr hyphen-de hyphen-es hyphen-it \
|
||||
hyphen-pt-pt hyphen-pt-br hyphen-nl hyphen-pl hyphen-ru \
|
||||
fontconfig \
|
||||
libcairo2 libpango-1.0-0 libpangoft2-1.0-0 libgdk-pixbuf-2.0-0 \
|
||||
libfreetype6 libfontconfig1 libx11-6 libxt6 libxext6 libxrender1 \
|
||||
libxtst6 libxi6 libxinerama1 libxkbcommon0 libsm6 libice6 \
|
||||
libglib2.0-0; \
|
||||
if [ "${INSTALL_CJK_FONTS}" = "true" ]; then \
|
||||
apt-get install -y --no-install-recommends fonts-noto-cjk; \
|
||||
fi; \
|
||||
apt-get remove --purge -y software-properties-common; \
|
||||
apt-get autoremove --purge -y; \
|
||||
rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
# Noble's hyphen-* packages skip LibreOffice registration; wire it up by hand
|
||||
# or the dicts never load.
|
||||
RUN set -eu; \
|
||||
mkdir -p /usr/share/myspell/dicts /usr/share/myspell/infos/ooo; \
|
||||
for f in /usr/share/hyphen/hyph_*_*.dic; do \
|
||||
base=$(basename "$f" .dic); \
|
||||
locale=${base#hyph_}; \
|
||||
lang=${locale%_*}; \
|
||||
country=${locale#*_}; \
|
||||
ln -sfn "../../hyphen/${base}.dic" "/usr/share/myspell/dicts/${base}.dic"; \
|
||||
printf "HYPH %s %s %s\n" "$lang" "$country" "$base" \
|
||||
> "/usr/share/myspell/infos/ooo/hyphen-${lang}-${country}"; \
|
||||
done
|
||||
|
||||
COPY --chmod=0644 docker/unoserver/fonts.conf /etc/fonts/conf.d/100-stirling.conf
|
||||
RUN fc-cache -f
|
||||
|
||||
# venv with --system-site-packages so it picks up distro python3-uno.
|
||||
RUN python3 -m venv /opt/unoserver-venv --system-site-packages && \
|
||||
/opt/unoserver-venv/bin/pip install --no-cache-dir "unoserver==${UNOSERVER_VERSION}" && \
|
||||
ln -sf /opt/unoserver-venv/bin/unoserver /usr/local/bin/unoserver && \
|
||||
ln -sf /opt/unoserver-venv/bin/unoconvert /usr/local/bin/unoconvert && \
|
||||
ln -sf /opt/unoserver-venv/bin/unoping /usr/local/bin/unoping && \
|
||||
find /opt/unoserver-venv -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
RUN rm -rf /usr/lib/libreoffice/share/gallery \
|
||||
/usr/lib/libreoffice/share/template \
|
||||
/usr/lib/libreoffice/share/wizards \
|
||||
/usr/lib/libreoffice/help \
|
||||
/usr/lib/libreoffice/share/extensions/wiki-publisher \
|
||||
/usr/lib/libreoffice/share/extensions/nlpsolver \
|
||||
/usr/share/doc/* /usr/share/man/* /usr/share/info/*
|
||||
|
||||
ARG PUID=1001
|
||||
ARG PGID=1001
|
||||
RUN groupadd -g "${PGID}" unoserver && \
|
||||
useradd -u "${PUID}" -g unoserver -m -d /home/unoserver -s /bin/bash unoserver && \
|
||||
mkdir -p "${UNOSERVER_PROFILE_DIR}" && \
|
||||
chown -R unoserver:unoserver "${UNOSERVER_PROFILE_DIR}" /home/unoserver
|
||||
|
||||
COPY --chmod=0755 docker/unoserver/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
COPY --chmod=0755 docker/unoserver/healthcheck.sh /usr/local/bin/healthcheck.sh
|
||||
|
||||
USER unoserver
|
||||
WORKDIR /home/unoserver
|
||||
|
||||
EXPOSE 2003
|
||||
|
||||
HEALTHCHECK --interval=15s --timeout=5s --retries=4 --start-period=45s \
|
||||
CMD /usr/local/bin/healthcheck.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/entrypoint.sh"]
|
||||
|
||||
LABEL org.opencontainers.image.title="Stirling-PDF Unoserver" \
|
||||
org.opencontainers.image.description="Pinned unoserver build for Stirling-PDF remote UNO mode" \
|
||||
org.opencontainers.image.source="https://github.com/Stirling-Tools/Stirling-PDF" \
|
||||
org.opencontainers.image.licenses="MIT" \
|
||||
org.opencontainers.image.vendor="Stirling-Tools"
|
||||
@@ -0,0 +1 @@
|
||||
1.0.0
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
|
||||
PORT="${UNOSERVER_PORT:-2003}"
|
||||
UNO_PORT="${UNOSERVER_UNO_PORT:-2002}"
|
||||
INTERFACE="${UNOSERVER_INTERFACE:-0.0.0.0}"
|
||||
CONVERSION_TIMEOUT="${UNOSERVER_CONVERSION_TIMEOUT:-1800}"
|
||||
RECYCLE_INTERVAL_SECONDS="${UNOSERVER_RECYCLE_INTERVAL_SECONDS:-0}"
|
||||
RECYCLE_INTERVAL_FLOOR=60
|
||||
PROFILE_DIR="${UNOSERVER_PROFILE_DIR:-/var/lib/unoserver/profile}"
|
||||
|
||||
log() { printf '%s %s\n' "[unoserver-entrypoint]" "$*" >&2; }
|
||||
|
||||
case "$PORT" in ''|*[!0-9]*) log "Invalid UNOSERVER_PORT='$PORT'"; exit 64 ;; esac
|
||||
case "$UNO_PORT" in ''|*[!0-9]*) log "Invalid UNOSERVER_UNO_PORT='$UNO_PORT'"; exit 64 ;; esac
|
||||
case "$CONVERSION_TIMEOUT" in ''|*[!0-9]*) log "Invalid UNOSERVER_CONVERSION_TIMEOUT='$CONVERSION_TIMEOUT'"; exit 64 ;; esac
|
||||
case "$RECYCLE_INTERVAL_SECONDS" in ''|*[!0-9]*) log "Invalid UNOSERVER_RECYCLE_INTERVAL_SECONDS='$RECYCLE_INTERVAL_SECONDS'"; exit 64 ;; esac
|
||||
|
||||
mkdir -p "$PROFILE_DIR"
|
||||
|
||||
start_xvfb() {
|
||||
if command -v Xvfb >/dev/null 2>&1 && [ -z "${DISPLAY:-}" ]; then
|
||||
Xvfb :99 -screen 0 1024x768x24 -ac +extension GLX +render -noreset >/dev/null 2>&1 &
|
||||
XVFB_PID=$!
|
||||
export DISPLAY=:99
|
||||
sleep 1
|
||||
log "Xvfb started (pid $XVFB_PID, DISPLAY=$DISPLAY)"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
trap '' TERM INT EXIT
|
||||
if [ -n "${UNOSERVER_PID:-}" ] && kill -0 "$UNOSERVER_PID" 2>/dev/null; then
|
||||
log "Stopping unoserver (pid $UNOSERVER_PID)"
|
||||
pkill -TERM -P "$UNOSERVER_PID" 2>/dev/null || true
|
||||
kill -TERM "$UNOSERVER_PID" 2>/dev/null || true
|
||||
wait "$UNOSERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${XVFB_PID:-}" ] && kill -0 "$XVFB_PID" 2>/dev/null; then
|
||||
kill -TERM "$XVFB_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup TERM INT EXIT
|
||||
|
||||
start_unoserver() {
|
||||
log "Starting unoserver on ${INTERFACE}:${PORT} (uno-port ${UNO_PORT}, timeout ${CONVERSION_TIMEOUT}s, profile ${PROFILE_DIR})"
|
||||
# Pass --user-installation as a plain path; unoserver 3.6 wraps it itself
|
||||
# and crashes if pre-wrapped as a file:// URI.
|
||||
unoserver \
|
||||
--interface "$INTERFACE" \
|
||||
--port "$PORT" \
|
||||
--uno-port "$UNO_PORT" \
|
||||
--user-installation "${PROFILE_DIR}" \
|
||||
--conversion-timeout "$CONVERSION_TIMEOUT" \
|
||||
2> >(grep --line-buffered -v "POST /RPC2" >&2) \
|
||||
&
|
||||
UNOSERVER_PID=$!
|
||||
}
|
||||
|
||||
# Wall-clock recycle to bound LibreOffice memory growth. wait -n is unreliable
|
||||
# here because the unoserver job is wrapped in a process substitution.
|
||||
recycle_supervisor() {
|
||||
if [ "$RECYCLE_INTERVAL_SECONDS" -le 0 ]; then
|
||||
log "Recycle disabled (UNOSERVER_RECYCLE_INTERVAL_SECONDS=0)"
|
||||
wait "$UNOSERVER_PID"
|
||||
return $?
|
||||
fi
|
||||
local interval="$RECYCLE_INTERVAL_SECONDS"
|
||||
if [ "$interval" -lt "$RECYCLE_INTERVAL_FLOOR" ]; then
|
||||
log "Clamping recycle interval ${interval}s up to floor ${RECYCLE_INTERVAL_FLOOR}s"
|
||||
interval="$RECYCLE_INTERVAL_FLOOR"
|
||||
fi
|
||||
log "Recycle enabled: restart every ${interval}s"
|
||||
while true; do
|
||||
local elapsed=0
|
||||
while [ "$elapsed" -lt "$interval" ]; do
|
||||
if ! kill -0 "$UNOSERVER_PID" 2>/dev/null; then
|
||||
wait "$UNOSERVER_PID"
|
||||
local rc=$?
|
||||
log "unoserver exited on its own (rc=$rc); not recycling"
|
||||
return "$rc"
|
||||
fi
|
||||
sleep 1
|
||||
elapsed=$((elapsed + 1))
|
||||
done
|
||||
log "Recycling unoserver (pid ${UNOSERVER_PID})"
|
||||
pkill -TERM -P "$UNOSERVER_PID" 2>/dev/null || true
|
||||
kill -TERM "$UNOSERVER_PID" 2>/dev/null || true
|
||||
for _ in 1 2 3 4 5; do
|
||||
kill -0 "$UNOSERVER_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
pkill -KILL -P "$UNOSERVER_PID" 2>/dev/null || true
|
||||
kill -KILL "$UNOSERVER_PID" 2>/dev/null || true
|
||||
wait "$UNOSERVER_PID" 2>/dev/null || true
|
||||
rm -rf "${PROFILE_DIR:?}"/* 2>/dev/null || true
|
||||
start_unoserver
|
||||
log "unoserver restarted (pid ${UNOSERVER_PID})"
|
||||
done
|
||||
}
|
||||
|
||||
start_xvfb
|
||||
start_unoserver
|
||||
recycle_supervisor
|
||||
@@ -0,0 +1,30 @@
|
||||
<?xml version='1.0'?>
|
||||
<!DOCTYPE fontconfig SYSTEM 'fonts.dtd'>
|
||||
<!-- Subpixel hinting + slight hint + LCD filter for headless LibreOffice. -->
|
||||
<fontconfig>
|
||||
<match target="font">
|
||||
<edit mode="assign" name="rgba">
|
||||
<const>rgb</const>
|
||||
</edit>
|
||||
</match>
|
||||
<match target="font">
|
||||
<edit mode="assign" name="hinting">
|
||||
<bool>true</bool>
|
||||
</edit>
|
||||
</match>
|
||||
<match target="font">
|
||||
<edit mode="assign" name="hintstyle">
|
||||
<const>hintslight</const>
|
||||
</edit>
|
||||
</match>
|
||||
<match target="font">
|
||||
<edit mode="assign" name="antialias">
|
||||
<bool>true</bool>
|
||||
</edit>
|
||||
</match>
|
||||
<match target="font">
|
||||
<edit mode="assign" name="lcdfilter">
|
||||
<const>lcddefault</const>
|
||||
</edit>
|
||||
</match>
|
||||
</fontconfig>
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
# Real RPC ping — a TCP socket can be bound while LibreOffice is wedged.
|
||||
set -eu
|
||||
|
||||
PORT="${UNOSERVER_PORT:-2003}"
|
||||
|
||||
if command -v unoping >/dev/null 2>&1; then
|
||||
unoping --host 127.0.0.1 --port "$PORT" >/dev/null 2>&1
|
||||
exit $?
|
||||
fi
|
||||
|
||||
exec timeout 2 bash -c "cat < /dev/null > /dev/tcp/127.0.0.1/${PORT}"
|
||||
+21
@@ -37,6 +37,22 @@ STIRLING_RAG_TOP_K=20
|
||||
# rather than chain more searches.
|
||||
STIRLING_RAG_MAX_SEARCHES=5
|
||||
|
||||
# Chunked reasoner settings: how big each per-worker slice is (in characters),
|
||||
# how many workers may run in parallel against the fast model, and how long
|
||||
# any single worker is allowed to wait for a response before being abandoned.
|
||||
# Worker timeouts protect gather_notes from upstream model stalls (which
|
||||
# otherwise hang at the provider's ~10 minute HTTP default); the affected
|
||||
# slice is dropped and the rest of the document still answers.
|
||||
STIRLING_CHUNKED_REASONER_CHARS_PER_SLICE=16000
|
||||
STIRLING_CHUNKED_REASONER_CONCURRENCY=10
|
||||
STIRLING_CHUNKED_REASONER_WORKER_TIMEOUT_SECONDS=60
|
||||
|
||||
# When the rendered slice notes would exceed this many characters, the
|
||||
# reasoner folds them hierarchically with fast-model calls until they fit.
|
||||
# This keeps the synthesis prompt under the model's context limit on long
|
||||
# documents (a 3000-page novel produces ~900k chars of raw notes).
|
||||
STIRLING_CHUNKED_REASONER_NOTES_CHAR_BUDGET=250000
|
||||
|
||||
# Upper bounds on PDF page text the engine will request per extraction round.
|
||||
STIRLING_MAX_PAGES=200
|
||||
STIRLING_MAX_CHARACTERS=200000
|
||||
@@ -51,3 +67,8 @@ STIRLING_LOG_LEVEL=INFO
|
||||
|
||||
# Path to log file. Rolls daily, keeps 1 backup. Leave empty for console only.
|
||||
STIRLING_LOG_FILE=
|
||||
|
||||
# Set true to log every outgoing httpx / Anthropic SDK request with timing.
|
||||
# Use when diagnosing worker stalls: a hung call shows a "Request" line with
|
||||
# no matching "Response" line. Noisy; leave off in normal use.
|
||||
STIRLING_HTTP_DEBUG=false
|
||||
|
||||
+24
-14
@@ -23,7 +23,7 @@ dev = [
|
||||
"anyio>=4.0.0",
|
||||
"datamodel-code-generator[ruff]>=0.26.0",
|
||||
"pytest>=8.0.0",
|
||||
"pyright>=1.1.408",
|
||||
"pyrefly==1.0.0",
|
||||
"referencing>=0.35.0",
|
||||
"ruff>=0.14.10",
|
||||
]
|
||||
@@ -51,21 +51,11 @@ select = [
|
||||
"W",
|
||||
"RUF100",
|
||||
"UP",
|
||||
"PYI", # flake8-pyi: flags deprecated typing constructs
|
||||
"FA", # flake8-future-annotations: flags missing future annotations imports
|
||||
"BLE", # flake8-blind-except: flags bare `except Exception`
|
||||
"PYI", # flake8-pyi: flags deprecated typing constructs
|
||||
"FA", # flake8-future-annotations: flags missing future annotations imports
|
||||
"BLE", # flake8-blind-except: flags bare `except Exception`
|
||||
]
|
||||
|
||||
[tool.pyright]
|
||||
pythonVersion = "3.13"
|
||||
|
||||
reportImportCycles = "warning"
|
||||
reportUnnecessaryCast = "warning"
|
||||
reportUnnecessaryTypeIgnoreComment = "warning"
|
||||
reportUnusedImport = "warning"
|
||||
reportUnknownParameterType = "warning"
|
||||
reportDeprecated = "warning"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
# ``tests`` is on the path so test modules can import shared helpers (e.g.
|
||||
@@ -74,3 +64,23 @@ pythonpath = ["src", "tests"]
|
||||
# Use importlib import mode so test directories don't need __init__.py files
|
||||
# and duplicate basenames (e.g. multiple test_routes.py) collect cleanly.
|
||||
addopts = "--import-mode=importlib"
|
||||
|
||||
[tool.pyrefly]
|
||||
python-version = "3.13.0"
|
||||
search-path = [
|
||||
"src",
|
||||
"tests",
|
||||
]
|
||||
# Disallow `type: ignore` comments to encourage accurate error codes to be used in ignore comments
|
||||
# Pyrefly won't flag `type: ignore [wrong-code]`, but it will flag `pyrefly: ignore [wrong-code]`
|
||||
enabled-ignores = ["pyrefly"]
|
||||
min-severity = "warn" # TODO: Change settings below to `warn` - https://github.com/facebook/pyrefly/issues/3370
|
||||
|
||||
[tool.pyrefly.errors]
|
||||
deprecated = "error"
|
||||
implicit-any = "error"
|
||||
redundant-cast = "error"
|
||||
unused-ignore = "error"
|
||||
# TODO: Add equivalents to:
|
||||
# - reportImportCycles
|
||||
# - reportUnusedImport
|
||||
|
||||
@@ -478,6 +478,7 @@ class MathAuditorAgent:
|
||||
|
||||
logger.info("[math-auditor-agent] extracting figures from page %d (%d chars)", folio.page, len(text))
|
||||
prompt = f"Page {folio.page + 1} text:\n{text}"
|
||||
figures: list[ExtractedFigure] = []
|
||||
try:
|
||||
result = await self._figure_extractor.run(prompt)
|
||||
figures = result.output.figures
|
||||
@@ -487,7 +488,6 @@ class MathAuditorAgent:
|
||||
folio.page,
|
||||
exc_info=True,
|
||||
)
|
||||
figures = []
|
||||
|
||||
logger.debug(
|
||||
"TOOL (extract_figures)\nArgs: %s\nResult: %s",
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from typing import Literal, overload
|
||||
from typing import Literal, Protocol, overload
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_ai import Agent
|
||||
@@ -72,6 +72,16 @@ class PdfEditSelectionAgent:
|
||||
return result.output
|
||||
|
||||
|
||||
class ParameterSelector(Protocol):
|
||||
async def select(
|
||||
self,
|
||||
request: PdfEditRequest,
|
||||
operation_plan: list[ToolEndpoint],
|
||||
operation_index: int,
|
||||
generated_steps: list[ToolOperationStep],
|
||||
) -> ParamToolModel: ...
|
||||
|
||||
|
||||
class PdfEditParameterSelector:
|
||||
def __init__(self, runtime: AppRuntime) -> None:
|
||||
self.runtime = runtime
|
||||
@@ -141,7 +151,7 @@ class PdfEditParameterSelector:
|
||||
class PdfEditAgent:
|
||||
def __init__(self, runtime: AppRuntime) -> None:
|
||||
self.runtime = runtime
|
||||
self.parameter_selector = PdfEditParameterSelector(runtime)
|
||||
self.parameter_selector: ParameterSelector = PdfEditParameterSelector(runtime)
|
||||
|
||||
async def orchestrate(self, request: OrchestratorRequest) -> PdfEditResponse:
|
||||
"""Entry point for the orchestrator delegate — adapts the orchestrator's
|
||||
|
||||
@@ -6,6 +6,7 @@ from pydantic_ai import Agent
|
||||
from pydantic_ai.output import NativeOutput
|
||||
|
||||
from stirling.agents.math_presentation import MathIntentClassifier, extract_math_verdict
|
||||
from stirling.agents.shared import ChunkedReasoner, WholeDocReaderCapability
|
||||
from stirling.contracts import (
|
||||
AiFile,
|
||||
EditPlanResponse,
|
||||
@@ -24,44 +25,46 @@ from stirling.contracts import (
|
||||
format_conversation_history,
|
||||
format_file_names,
|
||||
)
|
||||
from stirling.documents import RagCapability
|
||||
from stirling.models.agent_tool_models import AgentToolId, MathAuditorAgentParams
|
||||
from stirling.rag import RagCapability
|
||||
from stirling.services import AppRuntime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
PDF_QUESTION_SYSTEM_PROMPT = (
|
||||
"You answer questions about PDF documents by retrieving relevant content with the "
|
||||
"search_knowledge tool. Use it before answering. Do not guess or use outside knowledge.\n"
|
||||
"You answer questions about PDF documents using two retrieval tools:\n"
|
||||
"\n"
|
||||
"The search_knowledge tool has a finite call budget per run. When it is no longer "
|
||||
"available, answer from what you have already retrieved.\n"
|
||||
"1. search_knowledge(query) - returns the passages most semantically similar "
|
||||
"to the query. Use it for targeted lookups: a specific fact, a named section, "
|
||||
"a particular passage. Typically one or two calls is enough.\n"
|
||||
"\n"
|
||||
"2. read_full_document(query) - reads every page of the attached documents in "
|
||||
"parallel and returns notes relevant to the query. Use it when answering "
|
||||
"requires seeing the whole document end-to-end: summaries, aggregations "
|
||||
"(largest, shortest, count), comparisons across sections. It is more "
|
||||
"expensive than search_knowledge, so prefer search_knowledge when one or two "
|
||||
"passages would suffice.\n"
|
||||
"\n"
|
||||
"Pick the right tool, call it, then answer from what you got back. Do not "
|
||||
"guess or use outside knowledge.\n"
|
||||
"\n"
|
||||
"Guidelines:\n"
|
||||
"- Make targeted search_knowledge calls. Typically one or two is enough.\n"
|
||||
"- Answer from the retrieved text. If the retrieved content doesn't support a confident "
|
||||
"answer, return not_found.\n"
|
||||
"- For questions that would require reading the entire document end-to-end (e.g. "
|
||||
"'what's the shortest chapter', 'how many X are there'), return not_found.\n"
|
||||
"- Include a short list of evidence snippets (with page numbers where available) drawn "
|
||||
"from what search_knowledge returned.\n"
|
||||
"- If the retrieved content does not support a confident answer, return not_found.\n"
|
||||
"- Include a short list of evidence snippets (with page numbers where available) "
|
||||
"drawn from what the tools returned.\n"
|
||||
"\n"
|
||||
"Writing the not_found reason:\n"
|
||||
"- The reason is shown directly to the end user, so write it in plain, friendly "
|
||||
"language. One or two short sentences.\n"
|
||||
"- NEVER mention 'RAG', 'retrieval', 'chunks', 'search results', 'targeted search', "
|
||||
"'search_knowledge', or other implementation details.\n"
|
||||
"- Be honest about the actual limitation. For questions that require full-document "
|
||||
"analysis (shortest chapter, word counts, etc.), explain that the document is too "
|
||||
"long to analyse end-to-end: you can only look up specific passages, and that's "
|
||||
"not enough to compare every part of the document against every other.\n"
|
||||
"'search_knowledge', 'read_full_document', or other implementation details.\n"
|
||||
"- For questions where the answer just isn't in the document, say so directly: "
|
||||
"'I couldn't find that information in the document.'\n"
|
||||
"- Do not make it sound like you're choosing not to answer. Be clear that it's "
|
||||
"a genuine constraint."
|
||||
"- Do not make it sound like you're choosing not to answer."
|
||||
)
|
||||
|
||||
|
||||
_MATH_SYNTH_SYSTEM_PROMPT = (
|
||||
"You are given a math-audit Verdict (structured JSON) and the user's "
|
||||
"original question. Answer the question in plain prose using only "
|
||||
@@ -83,6 +86,9 @@ class PdfQuestionAgent:
|
||||
model_settings=runtime.fast_model_settings,
|
||||
)
|
||||
self._math_intent_classifier = MathIntentClassifier(runtime)
|
||||
# Shared across whole-doc-reader instances so the worker agent and
|
||||
# semaphore are constructed once and reused per request.
|
||||
self._chunked_reasoner = ChunkedReasoner(runtime)
|
||||
|
||||
async def handle(self, request: PdfQuestionRequest) -> PdfQuestionResponse:
|
||||
logger.info(
|
||||
@@ -95,7 +101,7 @@ class PdfQuestionAgent:
|
||||
logger.info("[pdf-question] missing ingestions: %s", [file.name for file in missing])
|
||||
return NeedIngestResponse(
|
||||
resume_with=SupportedCapability.PDF_QUESTION,
|
||||
reason="Some files have not been ingested into RAG yet.",
|
||||
reason="Some files have not been ingested yet.",
|
||||
files_to_ingest=missing,
|
||||
content_types=[PdfContentType.PAGE_TEXT],
|
||||
)
|
||||
@@ -145,23 +151,37 @@ class PdfQuestionAgent:
|
||||
async def _find_missing_files(self, files: list[AiFile]) -> list[AiFile]:
|
||||
missing: list[AiFile] = []
|
||||
for file in files:
|
||||
if not await self.runtime.rag_service.has_collection(file.id):
|
||||
if not await self.runtime.documents.has_collection(file.id):
|
||||
missing.append(file)
|
||||
return missing
|
||||
|
||||
async def _run_answer_agent(self, request: PdfQuestionRequest) -> PdfQuestionTerminalResponse:
|
||||
"""Drive a single smart-model agent with both retrieval tools.
|
||||
|
||||
The agent picks ``search_knowledge`` for targeted lookups and
|
||||
``read_full_document`` for whole-document questions. Removing the
|
||||
upstream classifier keeps that judgement in the same call that writes
|
||||
the answer, and lets the agent mix tools when the question warrants it.
|
||||
"""
|
||||
rag = RagCapability(
|
||||
rag_service=self.runtime.rag_service,
|
||||
documents=self.runtime.documents,
|
||||
collections=[file.id for file in request.files],
|
||||
top_k=self.runtime.settings.rag_default_top_k,
|
||||
max_searches=self.runtime.settings.rag_max_searches,
|
||||
)
|
||||
whole_doc = WholeDocReaderCapability(
|
||||
runtime=self.runtime,
|
||||
files=request.files,
|
||||
reasoner=self._chunked_reasoner,
|
||||
)
|
||||
agent = Agent(
|
||||
model=self.runtime.smart_model,
|
||||
output_type=NativeOutput([PdfQuestionAnswerResponse, PdfQuestionNotFoundResponse]),
|
||||
system_prompt=PDF_QUESTION_SYSTEM_PROMPT,
|
||||
instructions=rag.instructions,
|
||||
toolsets=[rag.toolset],
|
||||
# pydantic-ai accepts a list of (string-or-callable) instruction sources;
|
||||
# it resolves each at run time and concatenates them for the model.
|
||||
instructions=[rag.instructions, whole_doc.instructions],
|
||||
toolsets=[rag.toolset, whole_doc.toolset],
|
||||
model_settings=self.runtime.smart_model_settings,
|
||||
)
|
||||
prompt = self._build_prompt(request)
|
||||
@@ -184,5 +204,5 @@ class PdfQuestionAgent:
|
||||
f"Conversation history:\n{history}\n"
|
||||
f"Files: {format_file_names(request.files)}\n"
|
||||
f"Question: {request.question}\n"
|
||||
"Use search_knowledge to retrieve the relevant content, then answer."
|
||||
"Pick the right retrieval tool for this question, then answer from what it returns."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Reasoning utilities shared across agents."""
|
||||
|
||||
from stirling.agents.shared.chunked_reasoner import ChunkedReasoner, ChunkNotes
|
||||
from stirling.agents.shared.whole_doc_reader import WholeDocReaderCapability
|
||||
|
||||
__all__ = [
|
||||
"ChunkNotes",
|
||||
"ChunkedReasoner",
|
||||
"WholeDocReaderCapability",
|
||||
]
|
||||
@@ -0,0 +1,616 @@
|
||||
"""Chunked reasoning over long documents.
|
||||
|
||||
A reusable primitive for any agent that needs to answer a question that
|
||||
requires reading a whole document end-to-end. The document is split into
|
||||
character-budgeted chunks; each chunk is read by a parallel worker that
|
||||
extracts question-relevant notes; if the gathered notes overflow the
|
||||
synthesis context budget, the resulting notes are regrouped into fresh
|
||||
chunks and run through the same extractor again, until they fit.
|
||||
|
||||
Pages are tracked by the wrapper, never asked of the model: keeps the model
|
||||
output schema small and the page list authoritative.
|
||||
|
||||
Used wherever pure RAG retrieval is the wrong tool: aggregations ("largest
|
||||
number"), comparisons ("shortest chapter"), and full summaries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_ai.output import NativeOutput
|
||||
|
||||
from stirling.contracts import (
|
||||
WholeDocCompressionRound,
|
||||
WholeDocReadDone,
|
||||
WholeDocReadStarted,
|
||||
WholeDocSliceDone,
|
||||
)
|
||||
from stirling.contracts.documents import Page
|
||||
from stirling.models import ApiModel
|
||||
from stirling.services import AppRuntime, emit_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChunkNotes(ApiModel):
|
||||
"""Public-facing notes for a span of pages.
|
||||
|
||||
Returned to callers of :meth:`ChunkedReasoner.gather_notes` and to the
|
||||
inside of :meth:`ChunkedReasoner.reason`. The wrapper builds these from
|
||||
the model's :class:`_ExtractedNotes` output and a deterministic page list.
|
||||
"""
|
||||
|
||||
pages: list[int] = Field(description="Page numbers covered by these notes (1-indexed).")
|
||||
summary: str = Field(description="One- to three-sentence summary of the covered range.")
|
||||
relevant_excerpts: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Short verbatim quotes from the source content that bear on the user's question.",
|
||||
)
|
||||
facts: list[str] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"Concrete facts (numbers, names, dates, claims) the synthesiser may need. "
|
||||
"Includes candidate values for aggregation questions."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _ExtractedNotes(BaseModel):
|
||||
"""Model output for one extractor call.
|
||||
|
||||
No ``pages`` field: page numbers are mechanical aggregation the wrapper
|
||||
computes deterministically. Keeping them out of the schema saves output
|
||||
tokens for the bulkier excerpts/facts payload and prevents the model
|
||||
from misreporting page coverage.
|
||||
"""
|
||||
|
||||
summary: str = Field(description="One- to three-sentence summary of the supplied content.")
|
||||
relevant_excerpts: list[str] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"Short verbatim quotes drawn from the supplied content that bear on the question. "
|
||||
"Deduplicate; drop ones that don't bear on the question."
|
||||
),
|
||||
)
|
||||
facts: list[str] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"Distinct, deduplicated facts (numbers, names, dates, claims) needed to answer "
|
||||
"the question. For aggregation questions retain ALL candidate values across the "
|
||||
"supplied content so a later round can still pick the global winner."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Chunk:
|
||||
"""A unit of work for the extractor: content + the pages it covers + a fallback.
|
||||
|
||||
``content`` is the formatted text fed to the model: raw page text with
|
||||
``[Page N]`` markers in the first round, formatted prior-pass notes with
|
||||
``[Notes from pages A-B]`` markers in subsequent rounds. ``pages`` is
|
||||
attached to the resulting :class:`ChunkNotes` deterministically.
|
||||
|
||||
``fallback`` is the list of notes to keep if the extractor call fails. For
|
||||
raw page chunks it's empty (a failed slice has no pre-extracted notes to
|
||||
preserve). For chunks built from existing notes it's the input notes
|
||||
themselves, so a failure doesn't lose page coverage.
|
||||
"""
|
||||
|
||||
content: str
|
||||
pages: list[int]
|
||||
fallback: list[ChunkNotes]
|
||||
label: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RoundResult:
|
||||
"""Outcome of one extraction round.
|
||||
|
||||
``successes`` lets the loop detect rounds that made no forward progress
|
||||
(every chunk failed) and bail rather than spinning. ``slowest`` is the
|
||||
chunk with the longest successful extractor call this round, used for
|
||||
diagnostic log lines on the first round.
|
||||
"""
|
||||
|
||||
notes: list[ChunkNotes]
|
||||
successes: int
|
||||
slowest: tuple[str, float] | None
|
||||
|
||||
|
||||
def _page_range_label(pages: list[Page]) -> str:
|
||||
if not pages:
|
||||
return "pages=?"
|
||||
elif len(pages) == 1:
|
||||
return f"pages={pages[0].page_number}"
|
||||
else:
|
||||
return f"pages={pages[0].page_number}-{pages[-1].page_number}"
|
||||
|
||||
|
||||
def _note_range_label(notes: list[ChunkNotes]) -> str:
|
||||
"""Render a "pages=A-B" label for a group of already-extracted notes."""
|
||||
page_numbers = sorted({p for note in notes for p in note.pages})
|
||||
if not page_numbers:
|
||||
return "pages=?"
|
||||
if len(page_numbers) == 1:
|
||||
return f"pages={page_numbers[0]}"
|
||||
return f"pages={page_numbers[0]}-{page_numbers[-1]}"
|
||||
|
||||
|
||||
_EXTRACTOR_SYSTEM_PROMPT = (
|
||||
"You are reading content from a document - either raw page text or "
|
||||
"condensed notes from an earlier extraction pass - and your job is to "
|
||||
"produce a tight set of notes that captures everything relevant to the "
|
||||
"user's question. The same job runs many times in parallel across the "
|
||||
"document and may run again to consolidate notes into smaller batches, "
|
||||
"so be thorough: anything you skip cannot be recovered later.\n"
|
||||
"\n"
|
||||
"Output:\n"
|
||||
"- summary: 1-3 sentences covering the supplied content.\n"
|
||||
"- relevant_excerpts: short verbatim quotes from the supplied content "
|
||||
"that bear on the question. Deduplicate; drop quotes that don't help.\n"
|
||||
"- facts: concrete facts (numbers, names, dates, claims). Deduplicate; "
|
||||
"drop irrelevant ones. For aggregation questions (largest, smallest, "
|
||||
"count, total) retain ALL candidate values across the content so a "
|
||||
"later step can still pick the global winner.\n"
|
||||
"\n"
|
||||
"Stay grounded in the supplied content. Do not infer or fabricate "
|
||||
"anything that isn't already present. If nothing in the content is "
|
||||
"relevant to the question, return empty excerpts and facts and a short "
|
||||
"neutral summary."
|
||||
)
|
||||
|
||||
|
||||
class ChunkedReasoner:
|
||||
"""Run a question against a long document by chunking, mapping, and looping.
|
||||
|
||||
Two consumption styles:
|
||||
|
||||
* Tools that already have a synthesising LLM call upstream call
|
||||
:meth:`gather_notes` to get the structured notes and format them
|
||||
themselves with :meth:`format_notes`.
|
||||
* Callers that just want an answer call :meth:`reason`, which runs
|
||||
:meth:`gather_notes` and then a single synthesis call governed by the
|
||||
caller's ``answer_prompt`` and ``answer_type``.
|
||||
|
||||
Lifetime:
|
||||
Construct once per agent that uses it. The extractor agent is built
|
||||
at construction time and reused; the synthesis agent in :meth:`reason`
|
||||
is built per call because its output type is generic.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runtime: AppRuntime,
|
||||
*,
|
||||
chars_per_slice: int | None = None,
|
||||
concurrency: int | None = None,
|
||||
worker_timeout_seconds: float | None = None,
|
||||
notes_char_budget: int | None = None,
|
||||
) -> None:
|
||||
chars = chars_per_slice if chars_per_slice is not None else runtime.settings.chunked_reasoner_chars_per_slice
|
||||
conc = concurrency if concurrency is not None else runtime.settings.chunked_reasoner_concurrency
|
||||
timeout = (
|
||||
worker_timeout_seconds
|
||||
if worker_timeout_seconds is not None
|
||||
else runtime.settings.chunked_reasoner_worker_timeout_seconds
|
||||
)
|
||||
budget = (
|
||||
notes_char_budget if notes_char_budget is not None else runtime.settings.chunked_reasoner_notes_char_budget
|
||||
)
|
||||
if chars <= 0:
|
||||
raise ValueError("chars_per_slice must be positive")
|
||||
if conc <= 0:
|
||||
raise ValueError("concurrency must be positive")
|
||||
if timeout <= 0:
|
||||
raise ValueError("worker_timeout_seconds must be positive")
|
||||
if budget <= 0:
|
||||
raise ValueError("notes_char_budget must be positive")
|
||||
self._runtime = runtime
|
||||
self._chars_per_slice = chars
|
||||
self._worker_timeout_seconds = timeout
|
||||
self._notes_char_budget = budget
|
||||
self._semaphore = asyncio.Semaphore(conc)
|
||||
self._extractor: Agent[None, _ExtractedNotes] = Agent(
|
||||
model=runtime.fast_model,
|
||||
output_type=NativeOutput(_ExtractedNotes),
|
||||
system_prompt=_EXTRACTOR_SYSTEM_PROMPT,
|
||||
model_settings=runtime.fast_model_settings,
|
||||
)
|
||||
|
||||
async def gather_notes(self, pages: list[Page], question: str) -> list[ChunkNotes]:
|
||||
"""Return notes covering every page that fit the synthesis budget.
|
||||
|
||||
Worker failures are tolerated: surviving notes are returned. Returns
|
||||
an empty list only when every first-round chunk raises, which the
|
||||
caller can treat as a hard failure.
|
||||
|
||||
Progress events fire as each first-round chunk finishes (in completion
|
||||
order, not chunk order) carrying a monotonic ``completed`` counter so
|
||||
consumers can render "Read X of Y" with X advancing by exactly one
|
||||
per event. Subsequent compression rounds emit a single round-start
|
||||
event each.
|
||||
"""
|
||||
if not pages:
|
||||
raise ValueError("ChunkedReasoner.gather_notes requires at least one page")
|
||||
|
||||
chunks = [self._chunk_from_pages(slice_pages) for slice_pages in self._slice_pages(pages)]
|
||||
slice_total = len(chunks)
|
||||
logger.info(
|
||||
"[chunked-reasoner] question=%r pages=%d slices=%d",
|
||||
question,
|
||||
len(pages),
|
||||
slice_total,
|
||||
)
|
||||
await emit_progress(WholeDocReadStarted(question=question, pages=len(pages), slices=slice_total))
|
||||
|
||||
gather_start = time.perf_counter()
|
||||
notes = await self._run_chunks(chunks, question)
|
||||
|
||||
await emit_progress(
|
||||
WholeDocReadDone(
|
||||
completed=len(notes),
|
||||
slices=slice_total,
|
||||
duration_seconds=round(time.perf_counter() - gather_start, 2),
|
||||
)
|
||||
)
|
||||
return notes
|
||||
|
||||
async def _run_chunks(self, chunks: list[_Chunk], question: str) -> list[ChunkNotes]:
|
||||
"""Run chunks through the extractor, regrouping and looping until under budget.
|
||||
|
||||
The first round emits per-chunk progress events for streaming UIs;
|
||||
later rounds emit a single round-start event. Each round may produce
|
||||
fewer notes than chunks (every chunk maps to at most one consolidated
|
||||
note); when the rendered notes still exceed the budget, the survivors
|
||||
are regrouped into fresh chunks and the loop runs again.
|
||||
"""
|
||||
round_number = 0
|
||||
while True:
|
||||
chunks_in = len(chunks)
|
||||
result = await self._extract_chunks(chunks, question, round_number)
|
||||
|
||||
if result.slowest is not None:
|
||||
slow_label, slow_duration = result.slowest
|
||||
logger.info(
|
||||
"[chunked-reasoner] round %d: %d/%d chunks succeeded; slowest %s (%.1fs)",
|
||||
round_number,
|
||||
result.successes,
|
||||
chunks_in,
|
||||
slow_label,
|
||||
slow_duration,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"[chunked-reasoner] round %d: 0/%d chunks succeeded",
|
||||
round_number,
|
||||
chunks_in,
|
||||
)
|
||||
|
||||
rendered_size = self._rendered_notes_size(result.notes)
|
||||
if rendered_size <= self._notes_char_budget or len(result.notes) <= 1:
|
||||
if round_number > 0:
|
||||
logger.info(
|
||||
"[chunked-reasoner] compression done after %d round(s): %d notes, %d chars",
|
||||
round_number,
|
||||
len(result.notes),
|
||||
rendered_size,
|
||||
)
|
||||
return result.notes
|
||||
|
||||
if result.successes == 0:
|
||||
# No forward progress this round; further rounds would
|
||||
# reproduce the same shape. Return what we have.
|
||||
logger.warning(
|
||||
"[chunked-reasoner] round %d produced no successful extractions; bailing with %d notes",
|
||||
round_number,
|
||||
len(result.notes),
|
||||
)
|
||||
return result.notes
|
||||
|
||||
round_number += 1
|
||||
groups = self._group_notes_for_compression(result.notes)
|
||||
chunks = [self._chunk_from_notes(group) for group in groups]
|
||||
logger.info(
|
||||
"[chunked-reasoner] compression round %d: %d notes (%d chars) -> %d groups",
|
||||
round_number,
|
||||
len(result.notes),
|
||||
rendered_size,
|
||||
len(groups),
|
||||
)
|
||||
await emit_progress(
|
||||
WholeDocCompressionRound(
|
||||
round_number=round_number,
|
||||
notes_in=len(result.notes),
|
||||
groups=len(groups),
|
||||
)
|
||||
)
|
||||
|
||||
async def _extract_chunks(
|
||||
self,
|
||||
chunks: list[_Chunk],
|
||||
question: str,
|
||||
round_number: int,
|
||||
) -> _RoundResult:
|
||||
"""Run all chunks through the extractor in parallel; collect surviving notes.
|
||||
|
||||
Failures fall back to ``chunk.fallback`` (empty in the first round, so
|
||||
failures drop; populated in compression rounds, so failures preserve
|
||||
their input notes). The first round emits a
|
||||
:class:`WholeDocSliceDone` per successful completion in completion
|
||||
order, with a monotonic ``completed`` counter.
|
||||
|
||||
Returned notes are sorted by first page so downstream grouping packs
|
||||
document-adjacent content together regardless of which task happened
|
||||
to finish first.
|
||||
"""
|
||||
total = len(chunks)
|
||||
pending: dict[asyncio.Task[tuple[ChunkNotes, float]], _Chunk] = {
|
||||
asyncio.create_task(self._extract_chunk(chunk, question)): chunk for chunk in chunks
|
||||
}
|
||||
|
||||
notes: list[ChunkNotes] = []
|
||||
successes = 0
|
||||
slowest: tuple[str, float] | None = None
|
||||
completed = 0
|
||||
|
||||
try:
|
||||
while pending:
|
||||
done, _ = await asyncio.wait(pending.keys(), return_when=asyncio.FIRST_COMPLETED)
|
||||
for task in done:
|
||||
chunk = pending.pop(task)
|
||||
exc = task.exception()
|
||||
if exc is not None:
|
||||
if chunk.fallback:
|
||||
logger.warning(
|
||||
"[chunked-reasoner] chunk %s failed: %s; preserving %d input note(s)",
|
||||
chunk.label,
|
||||
exc,
|
||||
len(chunk.fallback),
|
||||
)
|
||||
notes.extend(chunk.fallback)
|
||||
else:
|
||||
logger.warning("[chunked-reasoner] chunk %s failed: %s", chunk.label, exc)
|
||||
continue
|
||||
extracted, duration = task.result()
|
||||
notes.append(extracted)
|
||||
successes += 1
|
||||
completed += 1
|
||||
if slowest is None or duration > slowest[1]:
|
||||
slowest = (chunk.label, duration)
|
||||
if round_number == 0:
|
||||
await emit_progress(
|
||||
WholeDocSliceDone(
|
||||
completed=completed,
|
||||
total=total,
|
||||
pages=chunk.label,
|
||||
duration_ms=int(duration * 1000),
|
||||
excerpts=len(extracted.relevant_excerpts),
|
||||
facts=len(extracted.facts),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
# On cancellation (typically a frontend disconnect propagating up
|
||||
# through the streaming orchestrator) the per-chunk model calls
|
||||
# would otherwise keep running to completion, billing tokens whose
|
||||
# results nobody is reading. Cancel and drain so the upstream
|
||||
# cancellation is the cancellation that matters.
|
||||
if pending:
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
await asyncio.gather(*pending.keys(), return_exceptions=True)
|
||||
|
||||
notes.sort(key=lambda n: n.pages[0] if n.pages else 0)
|
||||
return _RoundResult(notes=notes, successes=successes, slowest=slowest)
|
||||
|
||||
async def _extract_chunk(self, chunk: _Chunk, question: str) -> tuple[ChunkNotes, float]:
|
||||
"""Run the extractor on one chunk and attach the chunk's pages to the output."""
|
||||
try:
|
||||
extracted, duration = await self._run_extractor(chunk.content, question, chunk.label)
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
"[chunked-reasoner] chunk %s timed out (limit %.1fs)",
|
||||
chunk.label,
|
||||
self._worker_timeout_seconds,
|
||||
)
|
||||
raise
|
||||
logger.debug(
|
||||
"[chunked-reasoner] chunk %s: %d excerpt(s), %d fact(s) in %dms",
|
||||
chunk.label,
|
||||
len(extracted.relevant_excerpts),
|
||||
len(extracted.facts),
|
||||
int(duration * 1000),
|
||||
)
|
||||
return self._build_chunk_notes(extracted, chunk.pages), duration
|
||||
|
||||
async def _run_extractor(
|
||||
self,
|
||||
content: str,
|
||||
question: str,
|
||||
page_label: str,
|
||||
) -> tuple[_ExtractedNotes, float]:
|
||||
"""Inner primitive: run the extractor agent under semaphore + timeout."""
|
||||
prompt = self._build_extraction_prompt(content, question)
|
||||
async with self._semaphore:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await asyncio.wait_for(self._extractor.run(prompt), timeout=self._worker_timeout_seconds)
|
||||
except TimeoutError:
|
||||
duration = time.perf_counter() - start
|
||||
logger.debug(
|
||||
"[chunked-reasoner] extractor %s timed out after %dms",
|
||||
page_label,
|
||||
int(duration * 1000),
|
||||
)
|
||||
raise
|
||||
duration = time.perf_counter() - start
|
||||
return result.output, duration
|
||||
|
||||
def _chunk_from_pages(self, pages: list[Page]) -> _Chunk:
|
||||
"""Build a first-round chunk from a slice of raw pages."""
|
||||
return _Chunk(
|
||||
content="\n\n".join(f"[Page {p.page_number}]\n{p.text}" for p in pages),
|
||||
pages=[p.page_number for p in pages],
|
||||
fallback=[],
|
||||
label=_page_range_label(pages),
|
||||
)
|
||||
|
||||
def _chunk_from_notes(self, group: list[ChunkNotes]) -> _Chunk:
|
||||
"""Build a compression-round chunk from a group of prior-pass notes.
|
||||
|
||||
``fallback`` is the input group itself: if the extractor call fails,
|
||||
the originals stay in the working set so page coverage isn't lost.
|
||||
"""
|
||||
return _Chunk(
|
||||
content=self.format_notes(group),
|
||||
pages=sorted({p for note in group for p in note.pages}),
|
||||
fallback=group,
|
||||
label=_note_range_label(group),
|
||||
)
|
||||
|
||||
def _group_notes_for_compression(self, notes: list[ChunkNotes]) -> list[list[ChunkNotes]]:
|
||||
"""Pack consecutive notes into groups whose rendered size fits ``chars_per_slice``.
|
||||
|
||||
Each group becomes one compression-round chunk. Sized to match the
|
||||
first-round slice budget so the extractor sees roughly the same input
|
||||
footprint regardless of which round is running. Single notes that
|
||||
exceed the budget on their own become their own group.
|
||||
"""
|
||||
groups: list[list[ChunkNotes]] = []
|
||||
current: list[ChunkNotes] = []
|
||||
current_chars = 0
|
||||
for note in notes:
|
||||
note_chars = self._rendered_notes_size([note])
|
||||
if current and current_chars + note_chars > self._chars_per_slice:
|
||||
groups.append(current)
|
||||
current = []
|
||||
current_chars = 0
|
||||
current.append(note)
|
||||
current_chars += note_chars
|
||||
if current:
|
||||
groups.append(current)
|
||||
return groups
|
||||
|
||||
@staticmethod
|
||||
def _build_chunk_notes(extracted: _ExtractedNotes, pages: list[int]) -> ChunkNotes:
|
||||
"""Build a public ChunkNotes from the model's output and the wrapper's pages."""
|
||||
return ChunkNotes(
|
||||
pages=pages,
|
||||
summary=extracted.summary,
|
||||
relevant_excerpts=extracted.relevant_excerpts,
|
||||
facts=extracted.facts,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_extraction_prompt(content: str, question: str) -> str:
|
||||
"""Single prompt shape used for every round.
|
||||
|
||||
The system prompt explains the role; the user prompt just hands over
|
||||
the question and the content. Whether ``content`` is raw page text
|
||||
with ``[Page N]`` markers or formatted notes with ``[Notes from
|
||||
pages A-B]`` markers, the same instructions apply.
|
||||
"""
|
||||
return f"User question:\n{question}\n\nContent:\n{content}"
|
||||
|
||||
@staticmethod
|
||||
def _rendered_notes_size(notes: list[ChunkNotes]) -> int:
|
||||
"""Length in characters of what :meth:`format_notes` would produce."""
|
||||
return len(ChunkedReasoner.format_notes(notes))
|
||||
|
||||
async def reason[T: BaseModel](
|
||||
self,
|
||||
*,
|
||||
pages: list[Page],
|
||||
question: str,
|
||||
answer_prompt: str,
|
||||
answer_type: type[T],
|
||||
) -> T:
|
||||
"""Map over pages, then synthesise a structured answer of type ``T``.
|
||||
|
||||
Args:
|
||||
pages: Document pages in order.
|
||||
question: The user's question, passed to both workers and the
|
||||
synthesiser. Workers use it to decide what's relevant.
|
||||
answer_prompt: System prompt for the synthesis stage. Should
|
||||
instruct the model to answer ``question`` from the notes
|
||||
supplied. Owned by the caller because the answer's tone,
|
||||
format, and grounding rules are domain-specific.
|
||||
answer_type: Pydantic model describing the structured answer.
|
||||
|
||||
Returns:
|
||||
An instance of ``answer_type`` produced by the synthesis stage.
|
||||
"""
|
||||
notes = await self.gather_notes(pages, question)
|
||||
if not notes:
|
||||
raise RuntimeError("All chunked-reasoning workers failed; no notes to synthesise from")
|
||||
return await self._synthesise(question, notes, answer_prompt, answer_type)
|
||||
|
||||
@staticmethod
|
||||
def format_notes(notes: list[ChunkNotes]) -> str:
|
||||
"""Render notes as readable text for inclusion in another agent's tool result.
|
||||
|
||||
Order is preserved. Page numbers, summary, excerpts and facts are all
|
||||
emitted; empty sections are omitted.
|
||||
"""
|
||||
sections: list[str] = []
|
||||
for n in notes:
|
||||
page_label = (
|
||||
f"pages {n.pages[0]}-{n.pages[-1]}"
|
||||
if len(n.pages) > 1
|
||||
else f"page {n.pages[0]}"
|
||||
if n.pages
|
||||
else "unknown pages"
|
||||
)
|
||||
block = [f"[Notes from {page_label}]", f"Summary: {n.summary}"]
|
||||
if n.relevant_excerpts:
|
||||
block.append("Relevant excerpts:")
|
||||
block.extend(f"- {e}" for e in n.relevant_excerpts)
|
||||
if n.facts:
|
||||
block.append("Facts:")
|
||||
block.extend(f"- {f}" for f in n.facts)
|
||||
sections.append("\n".join(block))
|
||||
return "\n\n".join(sections)
|
||||
|
||||
def _slice_pages(self, pages: list[Page]) -> list[list[Page]]:
|
||||
"""Group consecutive pages into character-budgeted slices.
|
||||
|
||||
Page boundaries are preserved: a single page is never split across
|
||||
slices. If one page exceeds the budget on its own, it becomes its
|
||||
own slice.
|
||||
"""
|
||||
slices: list[list[Page]] = []
|
||||
current: list[Page] = []
|
||||
current_chars = 0
|
||||
for page in pages:
|
||||
if current and current_chars + page.char_count > self._chars_per_slice:
|
||||
slices.append(current)
|
||||
current = []
|
||||
current_chars = 0
|
||||
current.append(page)
|
||||
current_chars += page.char_count
|
||||
if current:
|
||||
slices.append(current)
|
||||
return slices
|
||||
|
||||
async def _synthesise[T: BaseModel](
|
||||
self,
|
||||
question: str,
|
||||
notes: list[ChunkNotes],
|
||||
answer_prompt: str,
|
||||
answer_type: type[T],
|
||||
) -> T:
|
||||
agent: Agent[None, T] = Agent(
|
||||
model=self._runtime.smart_model,
|
||||
output_type=NativeOutput(answer_type),
|
||||
system_prompt=answer_prompt,
|
||||
model_settings=self._runtime.smart_model_settings,
|
||||
)
|
||||
prompt = f"User question:\n{question}\n\nNotes from across the document:\n\n{self.format_notes(notes)}"
|
||||
result = await agent.run(prompt)
|
||||
return result.output
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Tool capability that lets an agent read whole documents end-to-end.
|
||||
|
||||
Companion to :class:`stirling.documents.RagCapability`. Where ``RagCapability``
|
||||
gives an agent targeted vector retrieval, this gives it map-style whole-document
|
||||
reading: every page is read in parallel by fast-model workers, and the
|
||||
question-relevant notes are returned for the agent to synthesise.
|
||||
|
||||
Use both capabilities together when the agent should pick its strategy:
|
||||
``search_knowledge`` for specific lookups, ``read_full_document`` for
|
||||
aggregations, comparisons, and summaries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from pydantic_ai import FunctionToolset, RunContext, ToolDefinition
|
||||
from pydantic_ai.toolsets import AbstractToolset
|
||||
|
||||
from stirling.agents.shared.chunked_reasoner import ChunkedReasoner
|
||||
from stirling.contracts import AiFile
|
||||
from stirling.services import AppRuntime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Cap on per-run calls. One pass already reads every page of every attached
|
||||
# document, so a second call is almost always the smart model second-guessing
|
||||
# itself on a near-identical query (and doubles wall-clock time for a sizeable
|
||||
# document). If a follow-up genuinely needs more, ``search_knowledge`` is the
|
||||
# right escape hatch. Configurable per-construction in case a future caller
|
||||
# can prove a real two-read use case; the default stays at 1.
|
||||
DEFAULT_MAX_READS = 1
|
||||
|
||||
|
||||
class WholeDocReaderCapability:
|
||||
"""Bundles instructions and the ``read_full_document`` toolset for agent injection.
|
||||
|
||||
Lifecycle: a ``WholeDocReaderCapability`` instance is intended to live for
|
||||
the duration of a single agent run.
|
||||
|
||||
The agent picks between this and :class:`RagCapability` per the tool
|
||||
descriptions: targeted retrieval vs whole-document reading.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runtime: AppRuntime,
|
||||
files: list[AiFile],
|
||||
*,
|
||||
reasoner: ChunkedReasoner | None = None,
|
||||
max_reads: int = DEFAULT_MAX_READS,
|
||||
) -> None:
|
||||
self._runtime = runtime
|
||||
self._files = files
|
||||
self._reasoner = reasoner if reasoner is not None else ChunkedReasoner(runtime)
|
||||
self._max_reads = max_reads
|
||||
self._read_count = 0
|
||||
toolset: FunctionToolset[None] = FunctionToolset()
|
||||
toolset.add_function(
|
||||
self._read_full_document,
|
||||
name="read_full_document",
|
||||
prepare=self._prepare_read_full_document,
|
||||
)
|
||||
self._toolset = toolset
|
||||
|
||||
@property
|
||||
def instructions(self) -> str:
|
||||
names = ", ".join(f.name for f in self._files) if self._files else "the attached documents"
|
||||
return (
|
||||
"You have a 'read_full_document' tool that reads every page of "
|
||||
f"{names} in parallel and returns notes relevant to a query. "
|
||||
"Use it when answering requires seeing the whole document end-to-end "
|
||||
"(summaries, aggregations, comparisons across sections). One call "
|
||||
"already reads everything; phrase the query to cover all the angles "
|
||||
"you need in a single pass. For follow-ups or specific lookups use "
|
||||
"'search_knowledge', which is cheaper and targeted."
|
||||
)
|
||||
|
||||
@property
|
||||
def toolset(self) -> AbstractToolset[None]:
|
||||
return self._toolset
|
||||
|
||||
async def _prepare_read_full_document(
|
||||
self,
|
||||
ctx: RunContext[None],
|
||||
tool_def: ToolDefinition,
|
||||
) -> ToolDefinition | None:
|
||||
"""Hide the tool from the agent's toolset once the per-run budget is spent.
|
||||
Mirrors the search_knowledge prepare callback."""
|
||||
if self._read_count >= self._max_reads:
|
||||
return None
|
||||
return tool_def
|
||||
|
||||
async def _read_full_document(self, query: str) -> str:
|
||||
"""Read every page of the attached documents and return notes relevant to the query.
|
||||
|
||||
Use this when answering needs the whole document end-to-end - summaries,
|
||||
aggregations like 'largest number' or 'shortest chapter', or comparisons
|
||||
across sections. Slow and expensive (one fast-model call per slice per
|
||||
document); prefer search_knowledge for targeted lookups.
|
||||
|
||||
Args:
|
||||
query: A focused description of what to extract from the documents,
|
||||
phrased so a worker reading just one slice can decide what's
|
||||
relevant to the user's question.
|
||||
|
||||
Returns:
|
||||
Per-document sections of structured notes (page numbers, summary,
|
||||
relevant excerpts, extracted facts), already ordered by page.
|
||||
"""
|
||||
self._read_count += 1
|
||||
if not self._files:
|
||||
return "No documents attached to read."
|
||||
|
||||
sections: list[str] = []
|
||||
for file in self._files:
|
||||
pages = await self._runtime.documents.read_pages(file.id)
|
||||
if not pages:
|
||||
logger.info(
|
||||
"[whole-doc-reader] no stored pages for %s (id=%s); skipping",
|
||||
file.name,
|
||||
file.id,
|
||||
)
|
||||
continue
|
||||
notes = await self._reasoner.gather_notes(pages, query)
|
||||
if not notes:
|
||||
continue
|
||||
sections.append(f"=== {file.name} ===\n{ChunkedReasoner.format_notes(notes)}")
|
||||
|
||||
if not sections:
|
||||
return "Could not read any document content."
|
||||
|
||||
logger.info(
|
||||
"[whole-doc-reader] read query=%r files=%d -> %d chars",
|
||||
query,
|
||||
len(self._files),
|
||||
sum(len(s) for s in sections),
|
||||
)
|
||||
return "\n\n".join(sections)
|
||||
@@ -19,13 +19,13 @@ from stirling.agents.pdf_comment import PdfCommentAgent
|
||||
from stirling.api.middleware import UserIdMiddleware
|
||||
from stirling.api.routes import (
|
||||
agent_draft_router,
|
||||
document_router,
|
||||
execution_router,
|
||||
ledger_router,
|
||||
orchestrator_router,
|
||||
pdf_comments_router,
|
||||
pdf_edit_router,
|
||||
pdf_question_router,
|
||||
rag_router,
|
||||
)
|
||||
from stirling.config import AppSettings, load_settings
|
||||
from stirling.contracts import HealthResponse
|
||||
@@ -57,7 +57,7 @@ async def lifespan(fast_api: FastAPI):
|
||||
if tracer_provider:
|
||||
Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider))
|
||||
yield
|
||||
await runtime.rag_service.close()
|
||||
await runtime.documents.close()
|
||||
if tracer_provider:
|
||||
tracer_provider.shutdown()
|
||||
|
||||
@@ -69,7 +69,7 @@ app.include_router(pdf_edit_router)
|
||||
app.include_router(pdf_question_router)
|
||||
app.include_router(agent_draft_router)
|
||||
app.include_router(execution_router)
|
||||
app.include_router(rag_router)
|
||||
app.include_router(document_router)
|
||||
app.include_router(ledger_router)
|
||||
app.include_router(pdf_comments_router)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ from stirling.agents import (
|
||||
)
|
||||
from stirling.agents.ledger import MathAuditorAgent
|
||||
from stirling.agents.pdf_comment import PdfCommentAgent
|
||||
from stirling.rag import RagService
|
||||
from stirling.documents import DocumentService
|
||||
from stirling.services import AppRuntime
|
||||
|
||||
|
||||
@@ -39,8 +39,8 @@ def get_execution_planning_agent(request: Request) -> ExecutionPlanningAgent:
|
||||
return request.app.state.execution_planning_agent
|
||||
|
||||
|
||||
def get_rag_service(request: Request) -> RagService:
|
||||
return request.app.state.runtime.rag_service
|
||||
def get_document_service(request: Request) -> DocumentService:
|
||||
return request.app.state.runtime.documents
|
||||
|
||||
|
||||
def get_math_auditor_agent(request: Request) -> MathAuditorAgent:
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
from .agent_drafts import router as agent_draft_router
|
||||
from .documents import router as document_router
|
||||
from .execution import router as execution_router
|
||||
from .ledger import router as ledger_router
|
||||
from .orchestrator import router as orchestrator_router
|
||||
from .pdf_comments import router as pdf_comments_router
|
||||
from .pdf_edit import router as pdf_edit_router
|
||||
from .pdf_questions import router as pdf_question_router
|
||||
from .rag import router as rag_router
|
||||
|
||||
__all__ = [
|
||||
"agent_draft_router",
|
||||
"document_router",
|
||||
"execution_router",
|
||||
"ledger_router",
|
||||
"orchestrator_router",
|
||||
"pdf_comments_router",
|
||||
"pdf_edit_router",
|
||||
"pdf_question_router",
|
||||
"rag_router",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from stirling.api.dependencies import get_document_service
|
||||
from stirling.contracts import (
|
||||
DeleteDocumentResponse,
|
||||
IngestDocumentRequest,
|
||||
IngestDocumentResponse,
|
||||
)
|
||||
from stirling.documents import DocumentService
|
||||
from stirling.models import FileId
|
||||
|
||||
router = APIRouter(prefix="/api/v1/documents", tags=["documents"])
|
||||
|
||||
|
||||
@router.post("", response_model=IngestDocumentResponse)
|
||||
async def ingest_document(
|
||||
request: IngestDocumentRequest,
|
||||
documents: Annotated[DocumentService, Depends(get_document_service)],
|
||||
) -> IngestDocumentResponse:
|
||||
"""Replace-ingest a document's content under ``document_id``.
|
||||
|
||||
Stores both representations in one shot:
|
||||
* embedded chunks for RAG search,
|
||||
* ordered page text for whole-document reading.
|
||||
Any previously-stored content for this document is removed first.
|
||||
"""
|
||||
pages = request.page_text or []
|
||||
chunks_indexed = await documents.ingest(
|
||||
collection=request.document_id,
|
||||
pages=pages,
|
||||
source=request.source,
|
||||
)
|
||||
return IngestDocumentResponse(document_id=request.document_id, chunks_indexed=chunks_indexed)
|
||||
|
||||
|
||||
@router.delete("/{document_id}", response_model=DeleteDocumentResponse)
|
||||
async def delete_document(
|
||||
document_id: FileId,
|
||||
documents: Annotated[DocumentService, Depends(get_document_service)],
|
||||
) -> DeleteDocumentResponse:
|
||||
"""Remove a document's content. Idempotent."""
|
||||
existed = await documents.has_collection(document_id)
|
||||
if existed:
|
||||
await documents.delete_collection(document_id)
|
||||
return DeleteDocumentResponse(document_id=document_id, deleted=existed)
|
||||
@@ -1,19 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, assert_never
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from stirling.agents import OrchestratorAgent
|
||||
from stirling.api.dependencies import get_orchestrator_agent
|
||||
from stirling.contracts import OrchestratorRequest, OrchestratorResponse
|
||||
from stirling.contracts import OrchestratorRequest, OrchestratorResponse, ProgressEvent
|
||||
from stirling.services import reset_progress_emitter, set_progress_emitter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cadence for keep-alive heartbeats on the streaming endpoint. Java forwards
|
||||
# them to the frontend as SSE comments; their job is to make every layer of
|
||||
# the connection visibly alive at this rhythm so disconnects surface within a
|
||||
# bounded window instead of waiting for the next progress event.
|
||||
HEARTBEAT_INTERVAL_SECONDS = 10.0
|
||||
|
||||
router = APIRouter(prefix="/api/v1/orchestrator", tags=["orchestrator"])
|
||||
|
||||
|
||||
@router.post("", response_model=OrchestratorResponse)
|
||||
@router.post("")
|
||||
async def orchestrate(
|
||||
request: OrchestratorRequest,
|
||||
agent: Annotated[OrchestratorAgent, Depends(get_orchestrator_agent)],
|
||||
) -> OrchestratorResponse:
|
||||
return await agent.handle(request)
|
||||
) -> StreamingResponse:
|
||||
"""Run the orchestrator and stream NDJSON events.
|
||||
|
||||
Each output line is a JSON object with an ``event`` field. ``progress``
|
||||
events arrive whenever an inner agent reports work (e.g. each
|
||||
chunked-reasoner slice completing); the final ``result`` event carries the
|
||||
typed orchestrator response. ``error`` events surface failures without
|
||||
breaking the connection. ``heartbeat`` events fire on a fixed cadence to
|
||||
keep idle connections visibly alive so disconnects propagate.
|
||||
|
||||
The stream itself is the liveness signal: as long as events flow, work is
|
||||
alive. Java consumes this with a long total timeout and treats line
|
||||
arrival as forward progress.
|
||||
"""
|
||||
return StreamingResponse(
|
||||
_OrchestratorStream(
|
||||
agent=agent,
|
||||
request=request,
|
||||
heartbeat_interval_seconds=HEARTBEAT_INTERVAL_SECONDS,
|
||||
).iterate(),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProgressFrame:
|
||||
event: ProgressEvent
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ResultFrame:
|
||||
response: OrchestratorResponse
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ErrorFrame:
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _HeartbeatFrame:
|
||||
"""No payload: a heartbeat exists only to push bytes through the pipe.
|
||||
|
||||
Without periodic traffic, a slow workflow phase (e.g. all extractor
|
||||
workers busy on long calls) leaves the engine writer, Java's SSE
|
||||
forwarder, and the frontend's fetch all silently waiting. A closed
|
||||
connection at any layer wouldn't surface until the next real event,
|
||||
which could be many tens of seconds away. Heartbeats bound that window
|
||||
to :data:`HEARTBEAT_INTERVAL_SECONDS`.
|
||||
"""
|
||||
|
||||
|
||||
type _StreamFrame = _ProgressFrame | _ResultFrame | _ErrorFrame | _HeartbeatFrame
|
||||
|
||||
|
||||
def _serialize_frame(frame: _StreamFrame) -> bytes:
|
||||
"""Render a frame as one NDJSON line."""
|
||||
match frame:
|
||||
case _ProgressFrame(event=event):
|
||||
body = {"event": "progress", **event.model_dump(mode="json")}
|
||||
case _ResultFrame(response=response):
|
||||
body = {"event": "result", "response": response.model_dump(mode="json")}
|
||||
case _ErrorFrame(message=message):
|
||||
body = {"event": "error", "message": message}
|
||||
case _HeartbeatFrame():
|
||||
body = {"event": "heartbeat"}
|
||||
case _:
|
||||
assert_never(frame)
|
||||
return (json.dumps(body) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
class _OrchestratorStream:
|
||||
"""Drives one streaming orchestrator request.
|
||||
|
||||
Owns the per-request queue and pumps progress events through it; the agent
|
||||
runs as a child task so its emissions and the streaming response interleave.
|
||||
A heartbeat task pushes keep-alive messages onto the same queue at a fixed
|
||||
cadence so the connection stays visibly alive between progress events.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
agent: OrchestratorAgent,
|
||||
request: OrchestratorRequest,
|
||||
heartbeat_interval_seconds: float,
|
||||
) -> None:
|
||||
self._agent = agent
|
||||
self._request = request
|
||||
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
||||
self._queue: asyncio.Queue[_StreamFrame | None] = asyncio.Queue()
|
||||
|
||||
async def iterate(self) -> AsyncIterator[bytes]:
|
||||
token = set_progress_emitter(self._emit_progress)
|
||||
agent_task = asyncio.create_task(self._run_agent())
|
||||
heartbeat_task = asyncio.create_task(self._emit_heartbeats())
|
||||
try:
|
||||
while True:
|
||||
frame = await self._queue.get()
|
||||
if frame is None:
|
||||
break
|
||||
yield _serialize_frame(frame)
|
||||
finally:
|
||||
reset_progress_emitter(token)
|
||||
await self._cancel_task(heartbeat_task)
|
||||
await self._cancel_task(agent_task)
|
||||
|
||||
async def _emit_progress(self, event: ProgressEvent) -> None:
|
||||
await self._queue.put(_ProgressFrame(event=event))
|
||||
|
||||
async def _emit_heartbeats(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(self._heartbeat_interval_seconds)
|
||||
await self._queue.put(_HeartbeatFrame())
|
||||
|
||||
async def _run_agent(self) -> None:
|
||||
try:
|
||||
response = await self._agent.handle(self._request)
|
||||
await self._queue.put(_ResultFrame(response=response))
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("orchestrator stream failed")
|
||||
await self._queue.put(_ErrorFrame(message=str(exc)))
|
||||
finally:
|
||||
await self._queue.put(None)
|
||||
|
||||
@staticmethod
|
||||
async def _cancel_task(task: asyncio.Task[None]) -> None:
|
||||
if task.done():
|
||||
return
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("background task failed during cancellation", exc_info=True)
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from stirling.api.dependencies import get_rag_service
|
||||
from stirling.contracts import (
|
||||
DeleteDocumentResponse,
|
||||
IngestDocumentRequest,
|
||||
IngestDocumentResponse,
|
||||
PdfContentType,
|
||||
)
|
||||
from stirling.models import FileId
|
||||
from stirling.rag import Document, RagService
|
||||
|
||||
router = APIRouter(prefix="/api/v1/rag", tags=["rag"])
|
||||
|
||||
|
||||
@router.post("/documents", response_model=IngestDocumentResponse)
|
||||
async def ingest_document(
|
||||
request: IngestDocumentRequest,
|
||||
rag: Annotated[RagService, Depends(get_rag_service)],
|
||||
) -> IngestDocumentResponse:
|
||||
"""Replace-ingest a document's content under ``document_id``.
|
||||
|
||||
Any previously-stored content for this document is removed and the
|
||||
provided content replaces it wholesale. All pages are chunked up front
|
||||
and then embedded in a single batched call so large documents (e.g. a
|
||||
500-page book) don't fan out into hundreds of embedding requests.
|
||||
"""
|
||||
await rag.delete_collection(request.document_id)
|
||||
|
||||
chunks: list[Document] = []
|
||||
if request.page_text:
|
||||
for page in request.page_text:
|
||||
if not page.text.strip():
|
||||
continue
|
||||
chunks.extend(
|
||||
rag.chunk_text(
|
||||
text=page.text,
|
||||
source=f"{request.source}:page:{page.page_number}",
|
||||
base_metadata={
|
||||
"page_number": str(page.page_number),
|
||||
"content_type": PdfContentType.PAGE_TEXT.value,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
indexed = await rag.index_documents(request.document_id, chunks) if chunks else 0
|
||||
return IngestDocumentResponse(document_id=request.document_id, chunks_indexed=indexed)
|
||||
|
||||
|
||||
@router.delete("/documents/{document_id}", response_model=DeleteDocumentResponse)
|
||||
async def delete_document(
|
||||
document_id: FileId,
|
||||
rag: Annotated[RagService, Depends(get_rag_service)],
|
||||
) -> DeleteDocumentResponse:
|
||||
"""Remove a document's content from RAG. Idempotent."""
|
||||
existed = await rag.has_collection(document_id)
|
||||
if existed:
|
||||
await rag.delete_collection(document_id)
|
||||
return DeleteDocumentResponse(document_id=document_id, deleted=existed)
|
||||
@@ -38,18 +38,38 @@ class AppSettings(BaseSettings):
|
||||
rag_default_top_k: int = Field(validation_alias="STIRLING_RAG_TOP_K")
|
||||
rag_max_searches: int = Field(validation_alias="STIRLING_RAG_MAX_SEARCHES")
|
||||
|
||||
# Chunked reasoner settings (whole-document map-reduce).
|
||||
chunked_reasoner_chars_per_slice: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_CHARS_PER_SLICE")
|
||||
chunked_reasoner_concurrency: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_CONCURRENCY")
|
||||
chunked_reasoner_worker_timeout_seconds: float = Field(
|
||||
validation_alias="STIRLING_CHUNKED_REASONER_WORKER_TIMEOUT_SECONDS"
|
||||
)
|
||||
# Maximum size, in characters, of the rendered notes block before the
|
||||
# reasoner folds slice notes hierarchically. The Anthropic context limit
|
||||
# is 200k tokens (~880k chars); we leave a generous margin for the
|
||||
# downstream agent's system prompt, history, tool definitions, and
|
||||
# response budget.
|
||||
chunked_reasoner_notes_char_budget: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_NOTES_CHAR_BUDGET")
|
||||
|
||||
max_pages: int = Field(validation_alias="STIRLING_MAX_PAGES")
|
||||
max_characters: int = Field(validation_alias="STIRLING_MAX_CHARACTERS")
|
||||
|
||||
log_level: str = Field(default="INFO", validation_alias="STIRLING_LOG_LEVEL")
|
||||
log_file: str = Field(default="", validation_alias="STIRLING_LOG_FILE")
|
||||
# When true, raises httpx + httpcore logger levels so every outgoing
|
||||
# model SDK call is logged with timing. Use to diagnose worker stalls:
|
||||
# a hung request shows the "Request: POST ..." line with no matching
|
||||
# response line, confirming the hang is transport-layer (not in our
|
||||
# code or the Anthropic SDK itself). Off by default — DEBUG-level
|
||||
# output is high-volume.
|
||||
http_debug: bool = Field(default=False, validation_alias="STIRLING_HTTP_DEBUG")
|
||||
|
||||
posthog_enabled: bool = Field(validation_alias="STIRLING_POSTHOG_ENABLED")
|
||||
posthog_api_key: str = Field(validation_alias="STIRLING_POSTHOG_API_KEY")
|
||||
posthog_host: str = Field(validation_alias="STIRLING_POSTHOG_HOST")
|
||||
|
||||
|
||||
def _configure_logging(level_name: str, log_file: str) -> None:
|
||||
def _configure_logging(level_name: str, log_file: str, http_debug: bool) -> None:
|
||||
"""Configure the ``stirling`` logger hierarchy."""
|
||||
level = logging.getLevelNamesMapping().get(level_name.upper())
|
||||
if level is None:
|
||||
@@ -83,11 +103,43 @@ def _configure_logging(level_name: str, log_file: str) -> None:
|
||||
fh.setLevel(level)
|
||||
root.addHandler(fh)
|
||||
|
||||
if http_debug:
|
||||
_enable_http_debug(formatter)
|
||||
|
||||
|
||||
def _enable_http_debug(formatter: logging.Formatter) -> None:
|
||||
"""Surface every httpx/httpcore call against the Anthropic API.
|
||||
|
||||
httpx emits one INFO line per request with the URL and final status,
|
||||
which is the most useful signal for diagnosing hung worker calls: a
|
||||
successful call shows "Request" then "Response" within a second or two;
|
||||
a hung one shows "Request" with no matching response until it's
|
||||
cancelled. httpcore at DEBUG drills down to TCP / HTTP/2 stream events
|
||||
if the user wants to see exactly where bytes stop flowing.
|
||||
|
||||
The ``stirling`` console handler is scoped to its own logger tree, so
|
||||
we attach a dedicated stream handler here. Without it, httpx records
|
||||
propagate to the root logger which has no handler in our setup and the
|
||||
output is silently dropped.
|
||||
"""
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
|
||||
for name, level in (("httpx", logging.INFO), ("httpcore", logging.DEBUG)):
|
||||
lg = logging.getLogger(name)
|
||||
lg.setLevel(level)
|
||||
# Idempotent: avoid stacking handlers on settings reload.
|
||||
if not any(getattr(h, "_stirling_http_debug", False) for h in lg.handlers):
|
||||
handler._stirling_http_debug = True # pyrefly: ignore [missing-attribute]
|
||||
lg.addHandler(handler)
|
||||
lg.propagate = False
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def load_settings() -> AppSettings:
|
||||
load_dotenv(ENV_FILE)
|
||||
load_dotenv(ENV_LOCAL_FILE, override=True)
|
||||
settings = AppSettings.model_validate({})
|
||||
_configure_logging(settings.log_level, settings.log_file)
|
||||
_configure_logging(settings.log_level, settings.log_file, settings.http_debug)
|
||||
return settings
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user