Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7500150fbd | ||
|
|
776749277c | ||
|
|
41b1b89fcb | ||
|
|
4d4e994562 | ||
|
|
2b118556f3 | ||
|
|
fbaff56d1c | ||
|
|
a1b15e0570 | ||
|
|
76549288a9 | ||
|
|
b5d0c4a5ed | ||
|
|
8bfcf6eb7e | ||
|
|
b019f9b570 | ||
|
|
d7df87b684 | ||
|
|
a84b375f5d | ||
|
|
52358c5bf9 | ||
|
|
c1e68c27c5 | ||
|
|
d4edff9059 | ||
|
|
80febc9993 | ||
|
|
c500c2fae7 | ||
|
|
6e4e3e138c | ||
|
|
13ca7bd52e | ||
|
|
456a96e15c | ||
|
|
7c8a46255b | ||
|
|
0a1b4ec173 | ||
|
|
b8d8f028c9 | ||
|
|
cd56367295 | ||
|
|
40a2d2844f | ||
|
|
f1332ebe4f | ||
|
|
f79968f336 | ||
|
|
df43e09eca | ||
|
|
cb0f335e8a | ||
|
|
38d06d3104 | ||
|
|
fe33378333 | ||
|
|
5944cd106b | ||
|
|
fd81bf4cf8 | ||
|
|
d23318cfa6 | ||
|
|
142544c9af | ||
|
|
99a5f2a1bc | ||
|
|
d4a3df79e5 | ||
|
|
863cad22bd | ||
|
|
c06657c8f9 | ||
|
|
376c9aeb31 | ||
|
|
532a80211f | ||
|
|
19377cc688 | ||
|
|
d06a367b87 | ||
|
|
b22ffb3b3b | ||
|
|
6319936720 | ||
|
|
ce6abe6e23 | ||
|
|
ece3562dc9 | ||
|
|
84d4455682 | ||
|
|
b9a7f2083b | ||
|
|
b36f3e0875 | ||
|
|
aca0be7b71 | ||
|
|
1c6e9e8158 | ||
|
|
e4379184b5 | ||
|
|
e0e20559b6 | ||
|
|
af694d506d | ||
|
|
d5806b56dd | ||
|
|
5ccb56da2d | ||
|
|
49dbf76670 | ||
|
|
0a7cd1185d | ||
|
|
e67bb77a59 | ||
|
|
16f589448d | ||
|
|
75ea3c9a1f | ||
|
|
ae1f905975 | ||
|
|
7529190587 | ||
|
|
b9f9f84907 | ||
|
|
68ec176719 | ||
|
|
d17c3f4fec | ||
|
|
783a51950f | ||
|
|
38ccea074c | ||
|
|
a7307ff393 | ||
|
|
72729e99c1 | ||
|
|
5fba2720f0 | ||
|
|
01a1ef8c44 | ||
|
|
8535c7e9ac | ||
|
|
cca3f42623 | ||
|
|
1b7ffcdbac | ||
|
|
67a0ca6110 | ||
|
|
8e4b2e2fc6 | ||
|
|
3c93457021 |
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.14.1
|
||||
pkgver=2.14.2
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.14.1
|
||||
pkgver=2.14.2
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
build: &build
|
||||
- build.gradle
|
||||
- app/(common|core|proprietary)/build.gradle
|
||||
- app/(common|core|proprietary|saas)/build.gradle
|
||||
- Taskfile.yml
|
||||
- .taskfiles/backend.yml
|
||||
|
||||
openapi: &openapi
|
||||
- *build
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
- app/(common|core|proprietary|saas)/src/main/java/**
|
||||
|
||||
docker-base: &docker-base
|
||||
- docker/base/Dockerfile
|
||||
@@ -23,9 +23,9 @@ docker: &docker
|
||||
- *docker-base
|
||||
|
||||
project: &project
|
||||
- app/(common|core|proprietary)/src/(main|test)/java/**
|
||||
- app/(common|core|proprietary|saas)/src/(main|test)/java/**
|
||||
- *build
|
||||
- "app/(common|core|proprietary)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*"
|
||||
- "app/(common|core|proprietary|saas)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*"
|
||||
- exampleYmlFiles/**
|
||||
- gradle/**
|
||||
- libs/**
|
||||
@@ -82,7 +82,7 @@ tauri: &tauri
|
||||
# tool surfaces it generates models from.
|
||||
engine: &engine
|
||||
- engine/**
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
- app/(common|core|proprietary|saas)/src/main/java/**
|
||||
- .github/workflows/ai-engine.yml
|
||||
- Taskfile.yml
|
||||
- .taskfiles/engine.yml
|
||||
|
||||
@@ -63,6 +63,7 @@ labels:
|
||||
files:
|
||||
- 'app/core/src/main/resources/static/.*'
|
||||
- 'app/proprietary/src/main/resources/static/.*'
|
||||
- 'app/saas/src/main/resources/static/.*'
|
||||
- 'frontend/**'
|
||||
- 'frontend/.*'
|
||||
- 'frontend/**/.*'
|
||||
@@ -83,6 +84,7 @@ labels:
|
||||
- 'app/common/src/main/java/.*.java'
|
||||
- 'app/proprietary/src/main/java/.*.java'
|
||||
- 'app/core/src/main/java/.*.java'
|
||||
- 'app/saas/src/main/java/.*.java'
|
||||
|
||||
- label: 'Back End'
|
||||
files:
|
||||
@@ -90,6 +92,9 @@ labels:
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/controller/.*'
|
||||
- 'app/core/src/main/resources/settings.yml.template'
|
||||
- 'app/core/src/main/resources/application.properties'
|
||||
- 'app/proprietary/src/main/resources/application-proprietary.properties'
|
||||
- 'app/saas/src/main/resources/application-dev.properties'
|
||||
- 'app/saas/src/main/resources/application-saas.properties'
|
||||
- 'app/core/src/main/resources/banner.txt'
|
||||
- 'app/core/src/main/resources/static/python/png_to_webp.py'
|
||||
- 'app/core/src/main/resources/static/python/split_photos.py'
|
||||
@@ -153,6 +158,7 @@ labels:
|
||||
- 'app/common/src/test/.*'
|
||||
- 'app/proprietary/src/test/.*'
|
||||
- 'app/core/src/test/.*'
|
||||
- 'app/saas/src/test/.*'
|
||||
- 'testing/.*'
|
||||
- '.github/workflows/scorecards.yml'
|
||||
- 'exampleYmlFiles/test_cicd.yml'
|
||||
@@ -171,3 +177,4 @@ labels:
|
||||
- 'app/common/build.gradle'
|
||||
- 'app/proprietary/build.gradle'
|
||||
- 'app/core/build.gradle'
|
||||
- 'app/saas/build.gradle'
|
||||
|
||||
@@ -256,7 +256,7 @@ jobs:
|
||||
|
||||
- name: Build and push V2 image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-image.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
|
||||
@@ -270,7 +270,7 @@ jobs:
|
||||
|
||||
- name: Build and push PR-specific image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
@@ -296,7 +296,7 @@ jobs:
|
||||
|
||||
- name: Build and push engine image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && needs.check-comment.outputs.enable_prototypes == 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: ./engine
|
||||
file: ./engine/Dockerfile
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
distribution: temurin
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Build and push frontend image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-frontend.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/frontend/Dockerfile
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
- name: Build and push backend image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-backend.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/backend/Dockerfile
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -510,6 +510,7 @@ jobs:
|
||||
# cargo output unsigned, so checking it produces false negatives.
|
||||
- name: Verify Windows Code Signature
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master') }}
|
||||
timeout-minutes: 15
|
||||
shell: pwsh
|
||||
run: |
|
||||
$allSigned = $true
|
||||
@@ -531,11 +532,26 @@ jobs:
|
||||
|
||||
# Extract MSI and verify the inner exe (the file that actually gets installed).
|
||||
# This is the critical check - AV flags the installed exe at runtime.
|
||||
# Use lessmsi, not `msiexec /a`: msiexec serializes on the global
|
||||
# _MSIExecute mutex and hangs forever on hosted runners when another
|
||||
# installer is busy. lessmsi reads MSI tables directly - no mutex, no service.
|
||||
$msi = $msiFiles[0].FullName
|
||||
$extractDir = Join-Path $env:RUNNER_TEMP "msi-verify"
|
||||
if (Test-Path $extractDir) { Remove-Item $extractDir -Recurse -Force }
|
||||
$proc = Start-Process msiexec.exe -ArgumentList '/a', $msi, '/qn', "TARGETDIR=$extractDir" -Wait -PassThru -NoNewWindow
|
||||
if ($proc.ExitCode -eq 0) {
|
||||
New-Item -ItemType Directory -Force -Path $extractDir | Out-Null
|
||||
|
||||
choco install lessmsi -y --no-progress --limit-output | Out-Null
|
||||
|
||||
# Bound the extraction and kill on hang (defence in depth over timeout-minutes).
|
||||
$proc = Start-Process lessmsi -ArgumentList 'x', "`"$msi`"", "`"$extractDir\`"" -PassThru -NoNewWindow
|
||||
if (-not $proc.WaitForExit(120000)) {
|
||||
try { $proc.Kill() } catch {}
|
||||
Write-Host "[ERROR] MSI extraction timed out after 120s"
|
||||
$allSigned = $false
|
||||
} elseif ($proc.ExitCode -ne 0) {
|
||||
Write-Host "[ERROR] Failed to extract MSI for verification (exit code: $($proc.ExitCode))"
|
||||
$allSigned = $false
|
||||
} else {
|
||||
$innerExe = Get-ChildItem -Path $extractDir -Filter "stirling-pdf.exe" -Recurse -File | Select-Object -First 1
|
||||
if ($innerExe) {
|
||||
$sig = Get-AuthenticodeSignature -FilePath $innerExe.FullName
|
||||
@@ -548,9 +564,6 @@ jobs:
|
||||
Write-Host "[ERROR] Could not find stirling-pdf.exe inside MSI"
|
||||
$allSigned = $false
|
||||
}
|
||||
} else {
|
||||
Write-Host "[ERROR] Failed to extract MSI for verification (exit code: $($proc.ExitCode))"
|
||||
$allSigned = $false
|
||||
}
|
||||
|
||||
if (-not $allSigned) {
|
||||
@@ -800,7 +813,11 @@ jobs:
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
|
||||
with:
|
||||
tag_name: v${{ needs.determine-matrix.outputs.version }}
|
||||
generate_release_notes: true
|
||||
# Don't regenerate/append notes on re-runs, and don't force this into the
|
||||
# "Latest" slot - leave the release body and latest marker as they are.
|
||||
generate_release_notes: false
|
||||
append_body: false
|
||||
make_latest: false
|
||||
fail_on_unmatched_files: true
|
||||
# Installers + updater payloads + manifest. .sig contents are embedded
|
||||
# in latest.json so the .sig files themselves are not uploaded.
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
|
||||
- name: Build and push base image
|
||||
id: build-push-base
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: docker/base
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -145,7 +145,7 @@ jobs:
|
||||
id: build-push-latest
|
||||
# Empty-tag guard: build-push-action errors when asked to push with no tags.
|
||||
if: env.RUN_MAIN_APP == 'true' && steps.meta.outputs.tags != ''
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Build and push Unified Dockerfile (fat variant)
|
||||
id: build-push-fat
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-fat.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -236,7 +236,7 @@ jobs:
|
||||
|
||||
- name: Build and push Unified Dockerfile (ultra-lite variant)
|
||||
id: build-push-lite
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-lite.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -365,7 +365,7 @@ jobs:
|
||||
- name: Build and push unoserver image
|
||||
id: build-push-unoserver
|
||||
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode != 'skip'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
@@ -186,7 +186,7 @@ jobs:
|
||||
# (multi-platform + gha cache) against the published base image.
|
||||
- name: Build ${{ matrix.docker-rev }} (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && inputs.docker-base-changed != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -259,7 +259,7 @@ jobs:
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Build and push test image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"ignoredFiles": [
|
||||
"frontend/editor/src-tauri/icons/icon.png"
|
||||
]
|
||||
}
|
||||
@@ -80,6 +80,12 @@ tasks:
|
||||
OPEN: '{{.OPEN | default ""}}'
|
||||
env:
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
# Dev-only browser-tab label so concurrent worktrees are distinguishable.
|
||||
# Only the worktree folder basename (e.g. "wt1") is exposed — never the
|
||||
# full path, hostname, or user. Consumed at dev-serve time by vite.config
|
||||
# and dropped from production builds.
|
||||
STIRLING_DEV_LABEL:
|
||||
sh: basename "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cmds:
|
||||
- npx vite editor --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}}
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ public class RuntimePathConfig {
|
||||
defaultWatchedFolders,
|
||||
watchedFoldersDirs,
|
||||
pipeline != null ? pipeline.getWatchedFoldersDir() : null);
|
||||
this.pipelineWatchedFoldersPath = this.pipelineWatchedFoldersPaths.getFirst();
|
||||
this.pipelineWatchedFoldersPath = this.pipelineWatchedFoldersPaths.get(0);
|
||||
this.pipelineFinishedFoldersPath =
|
||||
resolvePath(
|
||||
defaultFinishedFolders,
|
||||
|
||||
@@ -246,6 +246,14 @@ public class ApplicationProperties {
|
||||
* and paused runs are kept regardless of age.
|
||||
*/
|
||||
private int runExpiryMinutes = 30;
|
||||
|
||||
/**
|
||||
* Whether a policy S3 source's custom endpoint may resolve to a loopback, link-local, or
|
||||
* private address. Off by default so a user-supplied endpoint cannot be pointed at internal
|
||||
* services (e.g. the cloud metadata address); enable for a self-hosted MinIO or other
|
||||
* in-network object store.
|
||||
*/
|
||||
private boolean allowPrivateS3Endpoints = false;
|
||||
}
|
||||
|
||||
@Data
|
||||
|
||||
@@ -60,40 +60,54 @@ public class Provider {
|
||||
}
|
||||
|
||||
private UsernameAttribute validateUsernameAttribute(UsernameAttribute usernameAttribute) {
|
||||
return switch (name) {
|
||||
case "google" -> validateGoogleUsernameAttribute(usernameAttribute);
|
||||
case "github" -> validateGitHubUsernameAttribute(usernameAttribute);
|
||||
case "keycloak" -> validateKeycloakUsernameAttribute(usernameAttribute);
|
||||
default -> usernameAttribute;
|
||||
};
|
||||
switch (name) {
|
||||
case "google" -> {
|
||||
return validateGoogleUsernameAttribute(usernameAttribute);
|
||||
}
|
||||
case "github" -> {
|
||||
return validateGitHubUsernameAttribute(usernameAttribute);
|
||||
}
|
||||
case "keycloak" -> {
|
||||
return validateKeycloakUsernameAttribute(usernameAttribute);
|
||||
}
|
||||
default -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateKeycloakUsernameAttribute(
|
||||
UsernameAttribute usernameAttribute) {
|
||||
return switch (usernameAttribute) {
|
||||
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME, PREFERRED_USERNAME -> usernameAttribute;
|
||||
switch (usernameAttribute) {
|
||||
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME, PREFERRED_USERNAME -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
default ->
|
||||
throw new UnsupportedClaimException(
|
||||
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateGoogleUsernameAttribute(UsernameAttribute usernameAttribute) {
|
||||
return switch (usernameAttribute) {
|
||||
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME -> usernameAttribute;
|
||||
switch (usernameAttribute) {
|
||||
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
default ->
|
||||
throw new UnsupportedClaimException(
|
||||
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateGitHubUsernameAttribute(UsernameAttribute usernameAttribute) {
|
||||
return switch (usernameAttribute) {
|
||||
case LOGIN, EMAIL, NAME -> usernameAttribute;
|
||||
switch (usernameAttribute) {
|
||||
case LOGIN, EMAIL, NAME -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
default ->
|
||||
throw new UnsupportedClaimException(
|
||||
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -361,8 +361,8 @@ public class PdfMarkdownConverter {
|
||||
if (xs.isEmpty()) {
|
||||
return List.of(lines);
|
||||
}
|
||||
float minX = xs.getFirst();
|
||||
float maxX = xs.getLast();
|
||||
float minX = xs.get(0);
|
||||
float maxX = xs.get(xs.size() - 1);
|
||||
float splitAt = (minX + maxX) / 2f;
|
||||
float biggestGap = 0;
|
||||
for (int i = 1; i < xs.size(); i++) {
|
||||
@@ -492,7 +492,7 @@ public class PdfMarkdownConverter {
|
||||
|
||||
List<List<Line>> anchorGroups = new ArrayList<>();
|
||||
List<Line> current = new ArrayList<>();
|
||||
current.add(cands.getFirst());
|
||||
current.add(cands.get(0));
|
||||
for (int i = 1; i < cands.size(); i++) {
|
||||
float gap = cands.get(i - 1).y - cands.get(i).y;
|
||||
if (gap > splitThreshold) {
|
||||
@@ -513,8 +513,8 @@ public class PdfMarkdownConverter {
|
||||
if (anchors.size() < 2) {
|
||||
continue;
|
||||
}
|
||||
float top = anchors.getFirst().y;
|
||||
float bottom = anchors.getLast().y;
|
||||
float top = anchors.get(0).y;
|
||||
float bottom = anchors.get(anchors.size() - 1).y;
|
||||
|
||||
// Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the
|
||||
// run's vertical span, with a little slack below the last row) into the anchor above.
|
||||
@@ -674,8 +674,8 @@ public class PdfMarkdownConverter {
|
||||
float minGutter = Math.max(10f, charWidth * 2.5f);
|
||||
List<float[]> merged = new ArrayList<>();
|
||||
for (float[] band : columns) {
|
||||
if (!merged.isEmpty() && band[0] - merged.getLast()[1] < minGutter) {
|
||||
merged.getLast()[1] = band[1];
|
||||
if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) {
|
||||
merged.get(merged.size() - 1)[1] = band[1];
|
||||
} else {
|
||||
merged.add(new float[] {band[0], band[1]});
|
||||
}
|
||||
@@ -734,7 +734,7 @@ public class PdfMarkdownConverter {
|
||||
}
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(buildGfmRow(rows.getFirst(), widths, cols)).append('\n');
|
||||
sb.append(buildGfmRow(rows.get(0), widths, cols)).append('\n');
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
|
||||
@@ -910,8 +910,8 @@ public class PdfMarkdownConverter {
|
||||
}
|
||||
// Only merge a sentence continuation between two text paragraphs, never into/out of a
|
||||
// table.
|
||||
if (!(output.getLast() instanceof String last)
|
||||
|| !(pageItems.getFirst() instanceof String first)) {
|
||||
if (!(output.get(output.size() - 1) instanceof String last)
|
||||
|| !(pageItems.get(0) instanceof String first)) {
|
||||
return;
|
||||
}
|
||||
if (!first.isEmpty()
|
||||
@@ -932,13 +932,13 @@ public class PdfMarkdownConverter {
|
||||
for (Object e : elements) {
|
||||
if (e instanceof TableBlock tb
|
||||
&& !out.isEmpty()
|
||||
&& out.getLast() instanceof TableBlock prev
|
||||
&& out.get(out.size() - 1) instanceof TableBlock prev
|
||||
&& columnsMatch(flatten(prev.rows()), flatten(tb.rows()))) {
|
||||
List<List<Line>> merged = new ArrayList<>(prev.rows());
|
||||
List<List<Line>> tail = tb.rows();
|
||||
if (!tail.isEmpty()
|
||||
&& !prev.rows().isEmpty()
|
||||
&& rowText(tail.getFirst()).equals(rowText(prev.rows().getFirst()))) {
|
||||
&& rowText(tail.get(0)).equals(rowText(prev.rows().get(0)))) {
|
||||
tail = tail.subList(1, tail.size());
|
||||
}
|
||||
merged.addAll(tail);
|
||||
@@ -971,7 +971,7 @@ public class PdfMarkdownConverter {
|
||||
continue;
|
||||
}
|
||||
if (e instanceof TableBlock tb && !tb.rows().isEmpty()) {
|
||||
return rowText(tb.rows().getFirst());
|
||||
return rowText(tb.rows().get(0));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
/**
|
||||
* Thread-scoped correlation id for one automation run — a single pipeline, policy, or AI-workflow
|
||||
* execution over its input file(s).
|
||||
*
|
||||
* <p>Automations dispatch each tool step as a separate internal loopback POST via {@link
|
||||
* InternalApiClient}. The orchestrator opens a run scope around its dispatch loop; {@code
|
||||
* InternalApiClient} reads {@link #current()} and stamps it on every sub-step request as {@link
|
||||
* #RUN_ID_HEADER}. The SaaS PAYG interceptor uses that header so all sub-steps of ONE run group
|
||||
* into a single charge, while two <em>separate</em> runs that happen to touch identical bytes stay
|
||||
* distinct charges (the old content+time-window grouping merged them).
|
||||
*
|
||||
* <p>Sub-steps dispatch synchronously on the orchestrator's own thread (loopback {@code
|
||||
* RestTemplate}), so this ThreadLocal is visible to {@code InternalApiClient}. The id then crosses
|
||||
* to the receiving request thread via the HTTP header — never via this ThreadLocal.
|
||||
*
|
||||
* <p>No-op when the id is absent (a standalone tool call): the interceptor treats a missing run id
|
||||
* as "its own charge", which is exactly what a one-off call should be.
|
||||
*/
|
||||
public final class AutomationRunContext {
|
||||
|
||||
/** Header carrying the run id on internal sub-step dispatches. */
|
||||
public static final String RUN_ID_HEADER = "X-Stirling-Run-Id";
|
||||
|
||||
private static final ThreadLocal<String> CURRENT = new ThreadLocal<>();
|
||||
|
||||
private AutomationRunContext() {}
|
||||
|
||||
/**
|
||||
* Opens a run scope on the current thread. Returns an {@link AutoCloseable} that restores the
|
||||
* previously-active id (nesting-safe) — use in try-with-resources around the dispatch loop.
|
||||
*/
|
||||
public static Scope open(String runId) {
|
||||
String previous = CURRENT.get();
|
||||
CURRENT.set(runId);
|
||||
return () -> {
|
||||
if (previous == null) {
|
||||
CURRENT.remove();
|
||||
} else {
|
||||
CURRENT.set(previous);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** The run id active on this thread, or {@code null} when not inside a run scope. */
|
||||
public static String current() {
|
||||
return CURRENT.get();
|
||||
}
|
||||
|
||||
/** AutoCloseable whose {@link #close()} declares no checked exception. */
|
||||
public interface Scope extends AutoCloseable {
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.slf4j.MDC;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
@@ -60,6 +61,17 @@ public class InternalApiClient {
|
||||
*/
|
||||
public static final String AUTOMATION_HEADER = "X-Stirling-Automation";
|
||||
|
||||
/**
|
||||
* Header carrying the parent policy's name onto each sub-step dispatch, read from MDC key
|
||||
* {@link #POLICY_NAME_MDC_KEY} (set by the policy runner on the worker thread). Lets the audit
|
||||
* layer attribute a tool step to the policy that ran it, instead of showing it as a bare direct
|
||||
* call.
|
||||
*/
|
||||
public static final String POLICY_NAME_HEADER = "X-Stirling-Policy-Name";
|
||||
|
||||
/** MDC key the policy runner stamps with the running policy's name; forwarded as a header. */
|
||||
public static final String POLICY_NAME_MDC_KEY = "auditPolicyName";
|
||||
|
||||
private final ServletContext servletContext;
|
||||
private final UserServiceInterface userService;
|
||||
private final TempFileManager tempFileManager;
|
||||
@@ -111,6 +123,27 @@ public class InternalApiClient {
|
||||
// step inside a policy run must bill as AUTOMATION, not AI). Set unconditionally because
|
||||
// every caller of this dispatcher is an automation surface by design.
|
||||
headers.add(AUTOMATION_HEADER, "true");
|
||||
// Propagate the current automation run id (set by the orchestrator around its dispatch
|
||||
// loop) so the PAYG interceptor groups every sub-step of this one run into a single charge,
|
||||
// and never merges two separate runs that happen to touch identical bytes. Absent → the
|
||||
// receiving call is treated as standalone. See AutomationRunContext.
|
||||
String runId = AutomationRunContext.current();
|
||||
if (runId != null && !runId.isEmpty()) {
|
||||
headers.add(AutomationRunContext.RUN_ID_HEADER, runId);
|
||||
}
|
||||
|
||||
// Forward the parent policy name (set in MDC by the policy runner) so the audited sub-step
|
||||
// ties back to its policy. Single-line, length-capped: it becomes an HTTP header value.
|
||||
String policyName = MDC.get(POLICY_NAME_MDC_KEY);
|
||||
if (policyName != null && !policyName.isBlank()) {
|
||||
String safe = policyName.replaceAll("[\\r\\n]", " ").trim();
|
||||
if (safe.length() > 200) {
|
||||
safe = safe.substring(0, 200);
|
||||
}
|
||||
if (!safe.isEmpty()) {
|
||||
headers.add(POLICY_NAME_HEADER, safe);
|
||||
}
|
||||
}
|
||||
|
||||
// A no-file ai/tools call (e.g. create-pdf-from-html-agent) sends only string params, so
|
||||
// without this RestTemplate would use urlencoded instead of the multipart the controller
|
||||
@@ -119,7 +152,7 @@ public class InternalApiClient {
|
||||
boolean hasFilePart =
|
||||
body.values().stream()
|
||||
.flatMap(java.util.List::stream)
|
||||
.anyMatch(Resource.class::isInstance);
|
||||
.anyMatch(v -> v instanceof Resource);
|
||||
if (isAiTool && !hasFilePart) {
|
||||
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
|
||||
}
|
||||
|
||||
@@ -144,8 +144,10 @@ public class TempFileCleanupService {
|
||||
int directoriesDeletedCount = 0;
|
||||
for (Path directory : registry.getTempDirectories()) {
|
||||
try {
|
||||
if (Files.exists(directory)) {
|
||||
if (Files.exists(directory)
|
||||
&& shouldDeleteRegisteredDirectory(directory, maxAgeMillis)) {
|
||||
GeneralUtils.deleteDirectory(directory);
|
||||
registry.unregisterDirectory(directory);
|
||||
directoriesDeletedCount++;
|
||||
log.debug("Cleaned up temporary directory: {}", directory);
|
||||
}
|
||||
@@ -275,6 +277,21 @@ public class TempFileCleanupService {
|
||||
return totalDeletedCount.get();
|
||||
}
|
||||
|
||||
private boolean shouldDeleteRegisteredDirectory(Path directory, long maxAgeMillis) {
|
||||
if (maxAgeMillis <= 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
long currentTime = System.currentTimeMillis();
|
||||
long lastModified = Files.getLastModifiedTime(directory).toMillis();
|
||||
return (currentTime - lastModified) > maxAgeMillis;
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not check directory age, skipping cleanup: {}", directory, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Get the system temp directory path based on configuration or system property. */
|
||||
private Path getSystemTempPath() {
|
||||
String systemTempDir =
|
||||
|
||||
@@ -140,25 +140,20 @@ public class ChecksumUtils {
|
||||
|
||||
for (String algorithm : algorithms) {
|
||||
String key = algorithm; // keep original key for output
|
||||
Object digestOrChecksum =
|
||||
switch (algorithm.toUpperCase(Locale.ROOT)) {
|
||||
case "CRC32" -> new CRC32();
|
||||
case "ADLER32" -> new Adler32();
|
||||
default -> {
|
||||
try {
|
||||
// For MessageDigest, pass the original name (case-insensitive
|
||||
// per JCA)
|
||||
yield MessageDigest.getInstance(algorithm);
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new IllegalStateException(
|
||||
"Unsupported algorithm: " + algorithm, e);
|
||||
}
|
||||
}
|
||||
};
|
||||
if (digestOrChecksum instanceof Checksum checksum) {
|
||||
checksums.put(key, checksum);
|
||||
} else {
|
||||
digests.put(key, (MessageDigest) digestOrChecksum);
|
||||
switch (algorithm.toUpperCase(Locale.ROOT)) {
|
||||
case "CRC32":
|
||||
checksums.put(key, new CRC32());
|
||||
break;
|
||||
case "ADLER32":
|
||||
checksums.put(key, new Adler32());
|
||||
break;
|
||||
default:
|
||||
try {
|
||||
// For MessageDigest, pass the original name (case-insensitive per JCA)
|
||||
digests.put(key, MessageDigest.getInstance(algorithm));
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new IllegalStateException("Unsupported algorithm: " + algorithm, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -751,7 +751,7 @@ public class ExceptionUtils {
|
||||
String targetDescription;
|
||||
if (errorInfo.affectedPages() != null && !errorInfo.affectedPages().isEmpty()) {
|
||||
if (errorInfo.affectedPages().size() == 1) {
|
||||
targetDescription = "page " + errorInfo.affectedPages().getFirst();
|
||||
targetDescription = "page " + errorInfo.affectedPages().get(0);
|
||||
} else {
|
||||
targetDescription =
|
||||
"pages "
|
||||
@@ -848,7 +848,7 @@ public class ExceptionUtils {
|
||||
}
|
||||
|
||||
// Use the first page number, or null if none found
|
||||
Integer pageNumber = affectedPages.isEmpty() ? null : affectedPages.getFirst();
|
||||
Integer pageNumber = affectedPages.isEmpty() ? null : affectedPages.get(0);
|
||||
|
||||
return new GhostscriptErrorInfo(
|
||||
ErrorCode.GHOSTSCRIPT_PAGE_DRAWING,
|
||||
|
||||
@@ -114,7 +114,7 @@ public enum FormFieldTypeSupport {
|
||||
return;
|
||||
}
|
||||
|
||||
PDAnnotationWidget widget = checkBox.getWidgets().getFirst();
|
||||
PDAnnotationWidget widget = checkBox.getWidgets().get(0);
|
||||
|
||||
PDAppearanceCharacteristicsDictionary appearanceChars =
|
||||
widget.getAppearanceCharacteristics();
|
||||
|
||||
@@ -88,16 +88,28 @@ public class FormUtils {
|
||||
* text)
|
||||
*/
|
||||
public String detectFieldType(PDField field) {
|
||||
return switch (field) {
|
||||
case PDSignatureField ignored -> FIELD_TYPE_SIGNATURE;
|
||||
case PDPushButton ignored -> FIELD_TYPE_BUTTON;
|
||||
case PDTextField ignored -> FIELD_TYPE_TEXT;
|
||||
case PDCheckBox ignored -> FIELD_TYPE_CHECKBOX;
|
||||
case PDComboBox ignored -> FIELD_TYPE_COMBOBOX;
|
||||
case PDListBox ignored -> FIELD_TYPE_LISTBOX;
|
||||
case PDRadioButton ignored -> FIELD_TYPE_RADIO;
|
||||
case null, default -> FIELD_TYPE_TEXT;
|
||||
};
|
||||
if (field instanceof PDSignatureField) {
|
||||
return FIELD_TYPE_SIGNATURE;
|
||||
}
|
||||
if (field instanceof PDPushButton) {
|
||||
return FIELD_TYPE_BUTTON;
|
||||
}
|
||||
if (field instanceof PDTextField) {
|
||||
return FIELD_TYPE_TEXT;
|
||||
}
|
||||
if (field instanceof PDCheckBox) {
|
||||
return FIELD_TYPE_CHECKBOX;
|
||||
}
|
||||
if (field instanceof PDComboBox) {
|
||||
return FIELD_TYPE_COMBOBOX;
|
||||
}
|
||||
if (field instanceof PDListBox) {
|
||||
return FIELD_TYPE_LISTBOX;
|
||||
}
|
||||
if (field instanceof PDRadioButton) {
|
||||
return FIELD_TYPE_RADIO;
|
||||
}
|
||||
return FIELD_TYPE_TEXT;
|
||||
}
|
||||
|
||||
public List<FormFieldInfo> extractFormFields(PDDocument document) {
|
||||
@@ -571,17 +583,22 @@ public class FormUtils {
|
||||
continue;
|
||||
}
|
||||
String type = info.type();
|
||||
Object value =
|
||||
switch (type) {
|
||||
case FIELD_TYPE_CHECKBOX ->
|
||||
isChecked(info.value()) ? Boolean.TRUE : Boolean.FALSE;
|
||||
case FIELD_TYPE_LISTBOX ->
|
||||
info.multiSelect() ? new ArrayList<>() : safeDefault(info.value());
|
||||
case FIELD_TYPE_BUTTON, FIELD_TYPE_SIGNATURE -> null;
|
||||
default -> safeDefault(info.value());
|
||||
};
|
||||
if (value == null) {
|
||||
continue; // skip non-fillable
|
||||
Object value;
|
||||
switch (type) {
|
||||
case FIELD_TYPE_CHECKBOX:
|
||||
value = isChecked(info.value()) ? Boolean.TRUE : Boolean.FALSE;
|
||||
break;
|
||||
case FIELD_TYPE_LISTBOX:
|
||||
if (info.multiSelect()) {
|
||||
value = new ArrayList<>();
|
||||
} else {
|
||||
value = safeDefault(info.value());
|
||||
}
|
||||
break;
|
||||
case FIELD_TYPE_BUTTON, FIELD_TYPE_SIGNATURE:
|
||||
continue; // skip non-fillable
|
||||
default:
|
||||
value = safeDefault(info.value());
|
||||
}
|
||||
record.put(info.name(), value);
|
||||
}
|
||||
@@ -932,44 +949,44 @@ public class FormUtils {
|
||||
if (selection == null || selection.trim().isEmpty()) return null;
|
||||
List<String> filtered =
|
||||
filterChoiceSelections(List.of(selection), allowedOptions, fieldName);
|
||||
return filtered.isEmpty() ? null : filtered.getFirst();
|
||||
return filtered.isEmpty() ? null : filtered.get(0);
|
||||
}
|
||||
|
||||
private void applyValueToField(PDField field, String value, boolean strict) throws IOException {
|
||||
try {
|
||||
switch (field) {
|
||||
case PDTextField textField -> setTextValue(textField, value);
|
||||
case PDCheckBox checkBox -> {
|
||||
LinkedHashSet<String> candidateStates = collectCheckBoxStates(checkBox);
|
||||
boolean shouldCheck = shouldCheckBoxBeChecked(value, candidateStates);
|
||||
try {
|
||||
if (shouldCheck) {
|
||||
checkBox.check();
|
||||
} else {
|
||||
checkBox.unCheck();
|
||||
}
|
||||
} catch (IOException checkProblem) {
|
||||
log.warn(
|
||||
"Failed to set checkbox state for '{}': {}",
|
||||
field.getFullyQualifiedName(),
|
||||
checkProblem.getMessage(),
|
||||
checkProblem);
|
||||
if (strict) {
|
||||
throw checkProblem;
|
||||
}
|
||||
if (field instanceof PDTextField textField) {
|
||||
setTextValue(textField, value);
|
||||
} else if (field instanceof PDCheckBox checkBox) {
|
||||
LinkedHashSet<String> candidateStates = collectCheckBoxStates(checkBox);
|
||||
boolean shouldCheck = shouldCheckBoxBeChecked(value, candidateStates);
|
||||
try {
|
||||
if (shouldCheck) {
|
||||
checkBox.check();
|
||||
} else {
|
||||
checkBox.unCheck();
|
||||
}
|
||||
} catch (IOException checkProblem) {
|
||||
log.warn(
|
||||
"Failed to set checkbox state for '{}': {}",
|
||||
field.getFullyQualifiedName(),
|
||||
checkProblem.getMessage(),
|
||||
checkProblem);
|
||||
if (strict) {
|
||||
throw checkProblem;
|
||||
}
|
||||
}
|
||||
case PDRadioButton radioButton -> {
|
||||
if (value != null && !value.isBlank()) {
|
||||
radioButton.setValue(value);
|
||||
}
|
||||
} else if (field instanceof PDRadioButton radioButton) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
radioButton.setValue(value);
|
||||
}
|
||||
case PDChoice choiceField -> applyChoiceValue(choiceField, value);
|
||||
case PDPushButton ignored -> log.debug("Ignore Push button");
|
||||
case PDSignatureField ignored ->
|
||||
log.debug("Skipping signature field '{}'", field.getFullyQualifiedName());
|
||||
case null -> log.warn("Attempted to set value on null field");
|
||||
default -> field.setValue(value != null ? value : "");
|
||||
} else if (field instanceof PDChoice choiceField) {
|
||||
applyChoiceValue(choiceField, value);
|
||||
} else if (field instanceof PDPushButton) {
|
||||
log.debug("Ignore Push button");
|
||||
} else if (field instanceof PDSignatureField) {
|
||||
log.debug("Skipping signature field '{}'", field.getFullyQualifiedName());
|
||||
} else {
|
||||
field.setValue(value != null ? value : "");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
@@ -1289,42 +1306,37 @@ public class FormUtils {
|
||||
|
||||
List<String> resolveOptions(PDTerminalField field) {
|
||||
try {
|
||||
return switch (field) {
|
||||
case PDChoice choice -> {
|
||||
LinkedHashSet<String> allowed = new LinkedHashSet<>();
|
||||
List<String> exportValues = choice.getOptionsExportValues();
|
||||
List<String> displayValues = choice.getOptionsDisplayValues();
|
||||
if (field instanceof PDChoice choice) {
|
||||
LinkedHashSet<String> allowed = new LinkedHashSet<>();
|
||||
List<String> exportValues = choice.getOptionsExportValues();
|
||||
List<String> displayValues = choice.getOptionsDisplayValues();
|
||||
|
||||
if (exportValues != null) {
|
||||
exportValues.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.forEach(allowed::add);
|
||||
}
|
||||
if (displayValues != null) {
|
||||
displayValues.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.forEach(allowed::add);
|
||||
}
|
||||
yield new ArrayList<>(allowed);
|
||||
if (exportValues != null) {
|
||||
exportValues.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.forEach(allowed::add);
|
||||
}
|
||||
case PDRadioButton radio -> {
|
||||
List<String> exports = radio.getExportValues();
|
||||
yield exports != null && !exports.isEmpty()
|
||||
? new ArrayList<>(exports)
|
||||
: Collections.emptyList();
|
||||
if (displayValues != null) {
|
||||
displayValues.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.forEach(allowed::add);
|
||||
}
|
||||
case PDCheckBox checkBox -> {
|
||||
List<String> exports = checkBox.getExportValues();
|
||||
yield exports != null && !exports.isEmpty()
|
||||
? new ArrayList<>(exports)
|
||||
: Collections.emptyList();
|
||||
return new ArrayList<>(allowed);
|
||||
} else if (field instanceof PDRadioButton radio) {
|
||||
List<String> exports = radio.getExportValues();
|
||||
if (exports != null && !exports.isEmpty()) {
|
||||
return new ArrayList<>(exports);
|
||||
}
|
||||
case null, default -> Collections.emptyList();
|
||||
};
|
||||
} else if (field instanceof PDCheckBox checkBox) {
|
||||
List<String> exports = checkBox.getExportValues();
|
||||
if (exports != null && !exports.isEmpty()) {
|
||||
return new ArrayList<>(exports);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Failed to resolve options for field '{}': {}",
|
||||
@@ -1453,7 +1465,7 @@ public class FormUtils {
|
||||
|
||||
// Only check options for choice-type fields (combobox, listbox, radio)
|
||||
if (CHOICE_FIELD_TYPES.contains(type) && options != null && !options.isEmpty()) {
|
||||
String optionCandidate = cleanLabel(options.getFirst());
|
||||
String optionCandidate = cleanLabel(options.get(0));
|
||||
if (optionCandidate != null && !looksGeneric(optionCandidate)) {
|
||||
return optionCandidate;
|
||||
}
|
||||
@@ -1545,7 +1557,7 @@ public class FormUtils {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDAnnotationWidget widget = widgets.getFirst();
|
||||
PDAnnotationWidget widget = widgets.get(0);
|
||||
PDRectangle originalRectangle = cloneRectangle(widget.getRectangle());
|
||||
PDPage page = resolveWidgetPage(document, widget, null);
|
||||
if (page == null || originalRectangle == null) {
|
||||
@@ -2434,19 +2446,19 @@ public class FormUtils {
|
||||
|
||||
private static int firstWidgetPageIndex(FormFieldWithCoordinates f) {
|
||||
return (f.getWidgets() != null && !f.getWidgets().isEmpty())
|
||||
? f.getWidgets().getFirst().getPageIndex()
|
||||
? f.getWidgets().get(0).getPageIndex()
|
||||
: -1;
|
||||
}
|
||||
|
||||
private static float firstWidgetY(FormFieldWithCoordinates f) {
|
||||
return (f.getWidgets() != null && !f.getWidgets().isEmpty())
|
||||
? f.getWidgets().getFirst().getY()
|
||||
? f.getWidgets().get(0).getY()
|
||||
: 0;
|
||||
}
|
||||
|
||||
private static float firstWidgetX(FormFieldWithCoordinates f) {
|
||||
return (f.getWidgets() != null && !f.getWidgets().isEmpty())
|
||||
? f.getWidgets().getFirst().getX()
|
||||
? f.getWidgets().get(0).getX()
|
||||
: 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,27 +26,29 @@ import lombok.extern.slf4j.Slf4j;
|
||||
public class ImageProcessingUtils {
|
||||
|
||||
static BufferedImage convertColorType(BufferedImage sourceImage, String colorType) {
|
||||
return switch (colorType) {
|
||||
case "greyscale" -> {
|
||||
BufferedImage convertedImage =
|
||||
BufferedImage convertedImage;
|
||||
switch (colorType) {
|
||||
case "greyscale":
|
||||
convertedImage =
|
||||
new BufferedImage(
|
||||
sourceImage.getWidth(),
|
||||
sourceImage.getHeight(),
|
||||
BufferedImage.TYPE_BYTE_GRAY);
|
||||
convertedImage.getGraphics().drawImage(sourceImage, 0, 0, null);
|
||||
yield convertedImage;
|
||||
}
|
||||
case "blackwhite" -> {
|
||||
BufferedImage convertedImage =
|
||||
break;
|
||||
case "blackwhite":
|
||||
convertedImage =
|
||||
new BufferedImage(
|
||||
sourceImage.getWidth(),
|
||||
sourceImage.getHeight(),
|
||||
BufferedImage.TYPE_BYTE_BINARY);
|
||||
convertedImage.getGraphics().drawImage(sourceImage, 0, 0, null);
|
||||
yield convertedImage;
|
||||
}
|
||||
default -> sourceImage;
|
||||
};
|
||||
break;
|
||||
default: // full color
|
||||
convertedImage = sourceImage;
|
||||
break;
|
||||
}
|
||||
return convertedImage;
|
||||
}
|
||||
|
||||
public static byte[] getImageData(BufferedImage image) {
|
||||
|
||||
@@ -330,7 +330,7 @@ public class PDFToFile {
|
||||
|
||||
if (outputFiles.size() == 1) {
|
||||
// Return single output file
|
||||
File outputFile = outputFiles.getFirst();
|
||||
File outputFile = outputFiles.get(0);
|
||||
if ("txt:Text".equals(outputFormat)) {
|
||||
outputFormat = "txt";
|
||||
}
|
||||
|
||||
@@ -307,7 +307,7 @@ public class ProcessExecutor {
|
||||
boolean isQpdf =
|
||||
commandToRun != null
|
||||
&& !commandToRun.isEmpty()
|
||||
&& commandToRun.getFirst().contains("qpdf");
|
||||
&& commandToRun.get(0).contains("qpdf");
|
||||
|
||||
if (!outputLines.isEmpty()) {
|
||||
String outputMessage = String.join("\n", outputLines);
|
||||
@@ -370,7 +370,7 @@ public class ProcessExecutor {
|
||||
}
|
||||
|
||||
// Check if this is a UNO conversion by looking for unoconvert executable
|
||||
String executable = command.getFirst();
|
||||
String executable = command.get(0);
|
||||
if (executable != null) {
|
||||
// Extract basename from path for matching
|
||||
String basename = executable;
|
||||
@@ -504,7 +504,7 @@ public class ProcessExecutor {
|
||||
}
|
||||
|
||||
// Validate executable (first argument)
|
||||
String executable = command.getFirst();
|
||||
String executable = command.get(0);
|
||||
if (executable == null || executable.isBlank()) {
|
||||
throw new IllegalArgumentException("Command executable must not be empty");
|
||||
}
|
||||
|
||||
@@ -155,6 +155,7 @@ public class TempFileManager {
|
||||
if (directory != null && Files.isDirectory(directory)) {
|
||||
try {
|
||||
GeneralUtils.deleteDirectory(directory);
|
||||
registry.unregisterDirectory(directory);
|
||||
log.debug("Deleted temp directory: {}", directory.toString());
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to delete temp directory: {}", directory.toString(), e);
|
||||
|
||||
@@ -85,6 +85,18 @@ public class TempFileRegistry {
|
||||
return directory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a temporary directory from the registry.
|
||||
*
|
||||
* @param directory The directory to unregister
|
||||
*/
|
||||
public void unregisterDirectory(Path directory) {
|
||||
if (directory != null) {
|
||||
tempDirectories.remove(directory);
|
||||
log.debug("Unregistered temp directory: {}", directory.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a third-party temporary file that requires special handling.
|
||||
*
|
||||
|
||||
@@ -114,7 +114,7 @@ public class YamlHelper {
|
||||
|
||||
for (NodeTuple tuple : mappingNode.getValue()) {
|
||||
ScalarNode keyNode = (tuple.getKeyNode() instanceof ScalarNode sk) ? sk : null;
|
||||
if (keyNode == null || !keyNode.getValue().equals(keys.getFirst())) {
|
||||
if (keyNode == null || !keyNode.getValue().equals(keys.get(0))) {
|
||||
updatedTuples.add(tuple);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -176,11 +176,13 @@ class TempFileCleanupServiceMoreTest {
|
||||
class ScheduledCleanup {
|
||||
|
||||
@Test
|
||||
@DisplayName("deletes registered temp directories and reports counts")
|
||||
@DisplayName("deletes stale registered temp directories and reports counts")
|
||||
void deletesRegisteredDirectories() throws IOException {
|
||||
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(2);
|
||||
Path regDir = Files.createDirectories(tempDir.resolve("registeredDir"));
|
||||
Files.createFile(regDir.resolve("inside.txt"));
|
||||
Files.setLastModifiedTime(
|
||||
regDir, FileTime.fromMillis(System.currentTimeMillis() - 2L * 60 * 60 * 1000));
|
||||
Set<Path> dirs = new HashSet<>();
|
||||
dirs.add(regDir);
|
||||
when(registry.getTempDirectories()).thenReturn(dirs);
|
||||
@@ -193,6 +195,22 @@ class TempFileCleanupServiceMoreTest {
|
||||
verify(tempFileManager).cleanupOldTempFiles(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("keeps a fresh registered temp directory")
|
||||
void keepsFreshRegisteredDirectory() throws IOException {
|
||||
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(0);
|
||||
Path regDir = Files.createDirectories(tempDir.resolve("freshRegisteredDir"));
|
||||
Files.createFile(regDir.resolve("inside.txt"));
|
||||
Set<Path> dirs = new HashSet<>();
|
||||
dirs.add(regDir);
|
||||
when(registry.getTempDirectories()).thenReturn(dirs);
|
||||
lenient().when(registry.contains(any(File.class))).thenReturn(false);
|
||||
|
||||
withIsolatedUserHome(cleanupService::scheduledCleanup);
|
||||
|
||||
assertThat(Files.exists(regDir)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("skips a registered directory that no longer exists")
|
||||
void skipsMissingRegisteredDirectory() {
|
||||
|
||||
@@ -721,7 +721,7 @@ class PDFToFileTest {
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
List<String> args = invocation.getArgument(0);
|
||||
String outputPath = args.getLast();
|
||||
String outputPath = args.get(args.size() - 1);
|
||||
Files.write(Path.of(outputPath), "Fake DOCX content".getBytes());
|
||||
return mockExecutorResult;
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package stirling.software.SPDF.config;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.servlet.MultipartConfigFactory;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -21,11 +22,7 @@ import stirling.software.SPDF.controller.web.UploadLimitService;
|
||||
@Slf4j
|
||||
public class MultipartConfiguration {
|
||||
|
||||
private final UploadLimitService uploadLimitService;
|
||||
|
||||
public MultipartConfiguration(UploadLimitService uploadLimitService) {
|
||||
this.uploadLimitService = uploadLimitService;
|
||||
}
|
||||
@Autowired private UploadLimitService uploadLimitService;
|
||||
|
||||
/**
|
||||
* Creates MultipartConfigElement that respects fileUploadLimit from settings.yml or environment
|
||||
|
||||
@@ -294,25 +294,24 @@ public class BookletImpositionController {
|
||||
|
||||
// Apply rotation if needed (rotate about origin), then translate to keep in cell
|
||||
switch (rot) {
|
||||
case 90 -> {
|
||||
case 90:
|
||||
cs.transform(Matrix.getRotateInstance(Math.PI / 2, 0, 0));
|
||||
// After 90° CCW, the content spans x in [-r.getHeight(), 0] and y in [0,
|
||||
// r.getWidth()]
|
||||
cs.transform(Matrix.getTranslateInstance(0, -r.getWidth()));
|
||||
}
|
||||
case 180 -> {
|
||||
break;
|
||||
case 180:
|
||||
cs.transform(Matrix.getRotateInstance(Math.PI, 0, 0));
|
||||
cs.transform(Matrix.getTranslateInstance(-r.getWidth(), -r.getHeight()));
|
||||
}
|
||||
case 270 -> {
|
||||
break;
|
||||
case 270:
|
||||
cs.transform(Matrix.getRotateInstance(3 * Math.PI / 2, 0, 0));
|
||||
// After 270° CCW, the content spans x in [0, r.getHeight()] and y in
|
||||
// [-r.getWidth(), 0]
|
||||
cs.transform(Matrix.getTranslateInstance(-r.getHeight(), 0));
|
||||
}
|
||||
default -> {
|
||||
break;
|
||||
default:
|
||||
// 0°: no-op
|
||||
}
|
||||
}
|
||||
|
||||
// Reuse LayerUtility passed from caller
|
||||
|
||||
@@ -9,6 +9,7 @@ import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
@@ -57,6 +58,8 @@ import stirling.software.jpdfium.doc.PdfBookmarkEditor.BookmarkTree;
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class MergeController {
|
||||
|
||||
private static final Pattern QUOTE_WRAP_PATTERN = Pattern.compile("^\"|\"$");
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
@@ -158,6 +161,30 @@ public class MergeController {
|
||||
};
|
||||
}
|
||||
|
||||
private String[] parseClientFileIds(String clientFileIds) {
|
||||
if (clientFileIds == null || clientFileIds.trim().isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
try {
|
||||
String trimmed = clientFileIds.trim();
|
||||
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||
String inside = trimmed.substring(1, trimmed.length() - 1).trim();
|
||||
if (inside.isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
String[] parts = inside.split(",");
|
||||
String[] result = new String[parts.length];
|
||||
for (int i = 0; i < parts.length; i++) {
|
||||
result[i] = QUOTE_WRAP_PATTERN.matcher(parts[i].trim()).replaceAll("");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to parse client file IDs: {}", clientFileIds, e);
|
||||
}
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
private void addTableOfContents(PDDocument mergedDocument, MultipartFile[] files) {
|
||||
PDDocumentOutline outline = new PDDocumentOutline();
|
||||
mergedDocument.getDocumentCatalog().setDocumentOutline(outline);
|
||||
|
||||
@@ -125,14 +125,17 @@ public class UIDataController {
|
||||
pipelineConfigs.add(content);
|
||||
}
|
||||
|
||||
for (int i = 0; i < pipelineConfigs.size(); i++) {
|
||||
String config = pipelineConfigs.get(i);
|
||||
for (String config : pipelineConfigs) {
|
||||
Map<String, Object> jsonContent =
|
||||
objectMapper.readValue(
|
||||
config, new TypeReference<Map<String, Object>>() {});
|
||||
String name = (String) jsonContent.get("name");
|
||||
if (name == null || name.length() < 1) {
|
||||
String filename = jsonFiles.get(i).getFileName().toString();
|
||||
String filename =
|
||||
jsonFiles
|
||||
.get(pipelineConfigs.indexOf(config))
|
||||
.getFileName()
|
||||
.toString();
|
||||
name = filename.substring(0, filename.lastIndexOf('.'));
|
||||
}
|
||||
Map<String, String> configWithName = new HashMap<>();
|
||||
@@ -298,14 +301,20 @@ public class UIDataController {
|
||||
}
|
||||
|
||||
private static String getFormatFromExtension(String extension) {
|
||||
return switch (extension) {
|
||||
case "ttf" -> "truetype";
|
||||
case "woff" -> "woff";
|
||||
case "woff2" -> "woff2";
|
||||
case "eot" -> "embedded-opentype";
|
||||
case "svg" -> "svg";
|
||||
default -> "";
|
||||
};
|
||||
switch (extension) {
|
||||
case "ttf":
|
||||
return "truetype";
|
||||
case "woff":
|
||||
return "woff";
|
||||
case "woff2":
|
||||
return "woff2";
|
||||
case "eot":
|
||||
return "embedded-opentype";
|
||||
case "svg":
|
||||
return "svg";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,7 +187,7 @@ public class ConvertImgPDFController {
|
||||
}
|
||||
|
||||
if (webpFiles.size() == 1) {
|
||||
Path webpFilePath = webpFiles.getFirst();
|
||||
Path webpFilePath = webpFiles.get(0);
|
||||
byte[] webpBytes = Files.readAllBytes(webpFilePath);
|
||||
Files.deleteIfExists(tempFile);
|
||||
tempFile = null;
|
||||
|
||||
@@ -147,7 +147,7 @@ public class ConvertSvgToPDF {
|
||||
String outputFilename =
|
||||
filenames.isEmpty()
|
||||
? "combined_svgs.pdf"
|
||||
: GeneralUtils.generateFilename(filenames.getFirst(), "_combined.pdf");
|
||||
: GeneralUtils.generateFilename(filenames.get(0), "_combined.pdf");
|
||||
|
||||
log.info("Successfully combined {} SVGs into single PDF", sanitizedSvgs.size());
|
||||
|
||||
@@ -203,7 +203,7 @@ public class ConvertSvgToPDF {
|
||||
|
||||
try {
|
||||
if (convertedPdfs.size() == 1) {
|
||||
ConvertedPdf pdf = convertedPdfs.getFirst();
|
||||
ConvertedPdf pdf = convertedPdfs.get(0);
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
Files.write(tempOut.getPath(), pdf.content);
|
||||
@@ -218,7 +218,7 @@ public class ConvertSvgToPDF {
|
||||
filenames.isEmpty()
|
||||
? "converted_svgs.zip"
|
||||
: GeneralUtils.generateFilename(
|
||||
filenames.getFirst(), "_converted_svgs.zip");
|
||||
filenames.get(0), "_converted_svgs.zip");
|
||||
TempFile zipFile = createZipFromPdfs(convertedPdfs);
|
||||
return WebResponseUtils.zipFileToWebResponse(zipFile, zipFilename);
|
||||
} catch (IOException e) {
|
||||
|
||||
@@ -82,7 +82,7 @@ public class ExtractCSVController {
|
||||
if (csvEntries.isEmpty()) {
|
||||
return ResponseEntity.noContent().build();
|
||||
} else if (csvEntries.size() == 1) {
|
||||
return createCsvResponse(csvEntries.getFirst(), baseName);
|
||||
return createCsvResponse(csvEntries.get(0), baseName);
|
||||
} else {
|
||||
return createZipResponse(csvEntries, baseName);
|
||||
}
|
||||
|
||||
@@ -113,9 +113,7 @@ public class AutoRenameController {
|
||||
mergedLineInfos.sort(
|
||||
Comparator.comparing((LineInfo li) -> li.fontSize).reversed());
|
||||
String title =
|
||||
mergedLineInfos.isEmpty()
|
||||
? null
|
||||
: mergedLineInfos.getFirst().text;
|
||||
mergedLineInfos.isEmpty() ? null : mergedLineInfos.get(0).text;
|
||||
|
||||
return title != null
|
||||
? title
|
||||
|
||||
@@ -333,7 +333,7 @@ public class AutoSplitPdfController {
|
||||
}
|
||||
|
||||
if (!splitDocuments.isEmpty() && !isValidQrCode) {
|
||||
splitDocuments.getLast().addPage(document.getPage(page));
|
||||
splitDocuments.get(splitDocuments.size() - 1).addPage(document.getPage(page));
|
||||
} else if (page == 0) {
|
||||
PDDocument firstDocument = new PDDocument();
|
||||
firstDocument.addPage(document.getPage(page));
|
||||
|
||||
@@ -267,7 +267,7 @@ public class CompressController {
|
||||
if (references.isEmpty()) continue;
|
||||
|
||||
// Get the first instance of this image
|
||||
PDImageXObject originalImage = getOriginalImage(doc, references.getFirst());
|
||||
PDImageXObject originalImage = getOriginalImage(doc, references.get(0));
|
||||
|
||||
// Track original size
|
||||
int originalSize = (int) originalImage.getCOSObject().getLength();
|
||||
@@ -1167,7 +1167,7 @@ public class CompressController {
|
||||
List<ImageReference> references = entry.getValue();
|
||||
if (references.isEmpty()) continue;
|
||||
|
||||
PDImageXObject originalImage = getOriginalImage(doc, references.getFirst());
|
||||
PDImageXObject originalImage = getOriginalImage(doc, references.get(0));
|
||||
|
||||
int originalSize = (int) originalImage.getCOSObject().getLength();
|
||||
stats.totalOriginalBytes += originalSize;
|
||||
|
||||
@@ -211,7 +211,7 @@ public class ExtractImageScansController {
|
||||
} else {
|
||||
|
||||
// Return the processed image as a response
|
||||
byte[] imageBytes = processedImageBytes.getFirst();
|
||||
byte[] imageBytes = processedImageBytes.get(0);
|
||||
finalOutput = tempFileManager.createManagedTempFile(".png");
|
||||
try (OutputStream out = Files.newOutputStream(finalOutput.getPath())) {
|
||||
out.write(imageBytes);
|
||||
|
||||
@@ -10,6 +10,7 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
@@ -27,6 +28,7 @@ import stirling.software.SPDF.model.PipelineConfig;
|
||||
import stirling.software.SPDF.model.PipelineOperation;
|
||||
import stirling.software.SPDF.model.PipelineResult;
|
||||
import stirling.software.SPDF.service.ApiDocService;
|
||||
import stirling.software.common.service.AutomationRunContext;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.ZipExtractionUtils;
|
||||
@@ -71,6 +73,17 @@ public class PipelineProcessor {
|
||||
|
||||
PipelineResult runPipelineAgainstFiles(List<Resource> outputFiles, PipelineConfig config)
|
||||
throws Exception {
|
||||
// One pipeline execution = one automation run. Scope a run id so every tool sub-step
|
||||
// dispatched via InternalApiClient groups into a single charge on the SaaS billing side
|
||||
// (see AutomationRunContext); pipeline steps run synchronously on this thread.
|
||||
try (AutomationRunContext.Scope ignored =
|
||||
AutomationRunContext.open(UUID.randomUUID().toString())) {
|
||||
return runPipelineAgainstFilesInternal(outputFiles, config);
|
||||
}
|
||||
}
|
||||
|
||||
private PipelineResult runPipelineAgainstFilesInternal(
|
||||
List<Resource> outputFiles, PipelineConfig config) throws Exception {
|
||||
PipelineResult result = new PipelineResult();
|
||||
|
||||
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
|
||||
|
||||
@@ -404,7 +404,7 @@ public class CertSignController {
|
||||
PDAcroForm acroForm = new PDAcroForm(doc);
|
||||
doc.getDocumentCatalog().setAcroForm(acroForm);
|
||||
PDSignatureField signatureField = new PDSignatureField(acroForm);
|
||||
PDAnnotationWidget widget = signatureField.getWidgets().getFirst();
|
||||
PDAnnotationWidget widget = signatureField.getWidgets().get(0);
|
||||
List<PDField> acroFormFields = acroForm.getFields();
|
||||
acroForm.setSignaturesExist(true);
|
||||
acroForm.setAppendOnly(true);
|
||||
|
||||
@@ -136,11 +136,17 @@ public class RedactController {
|
||||
+ "Users can provide text patterns to redact, with options for regex and whole word matching. "
|
||||
+ "Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<Resource> redactPdf(@ModelAttribute RedactPdfRequest request) {
|
||||
String[] listOfText = request.getListOfText().split("\n");
|
||||
String rawListOfText = request.getListOfText();
|
||||
boolean useRegex = Boolean.TRUE.equals(request.getUseRegex());
|
||||
boolean wholeWordSearchBool = Boolean.TRUE.equals(request.getWholeWordSearch());
|
||||
|
||||
if (listOfText.length == 0 || (listOfText.length == 1 && listOfText[0].trim().isEmpty())) {
|
||||
if (rawListOfText == null || rawListOfText.trim().isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.patterns", "No text patterns provided for redaction");
|
||||
}
|
||||
|
||||
String[] listOfText = rawListOfText.split("\n");
|
||||
if (listOfText.length == 1 && listOfText[0].trim().isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.patterns", "No text patterns provided for redaction");
|
||||
}
|
||||
|
||||
@@ -634,7 +634,7 @@ class RedactExecuteService {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(extractor.getLineBoxes(), pageWidth);
|
||||
if (layout.columnCount() > 1) {
|
||||
float[] g = layout.gutters().getFirst();
|
||||
float[] g = layout.gutters().get(0);
|
||||
log.info(
|
||||
"[redact/execute] page {} layout: 2 cols, gutter x=[{}, {}]",
|
||||
pageIdx + 1,
|
||||
|
||||
@@ -60,7 +60,7 @@ public class RemoveCertSignController {
|
||||
// Remove signature fields safely
|
||||
List<PDField> fieldsToRemove =
|
||||
acroForm.getFields().stream()
|
||||
.filter(PDSignatureField.class::isInstance)
|
||||
.filter(field -> field instanceof PDSignatureField)
|
||||
.toList();
|
||||
|
||||
if (!fieldsToRemove.isEmpty()) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package stirling.software.SPDF.controller.web;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -12,11 +13,7 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
@Slf4j
|
||||
public class UploadLimitService {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public UploadLimitService(ApplicationProperties applicationProperties) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
@Autowired private ApplicationProperties applicationProperties;
|
||||
|
||||
public long getUploadLimit() {
|
||||
String raw =
|
||||
|
||||
@@ -2688,7 +2688,7 @@ public class PdfJsonConversionService {
|
||||
|
||||
// Find which page the field is on
|
||||
PDAnnotationWidget widget =
|
||||
field.getWidgets().isEmpty() ? null : field.getWidgets().getFirst();
|
||||
field.getWidgets().isEmpty() ? null : field.getWidgets().get(0);
|
||||
if (widget != null) {
|
||||
PDPage fieldPage = widget.getPage();
|
||||
if (fieldPage != null) {
|
||||
@@ -3164,7 +3164,7 @@ public class PdfJsonConversionService {
|
||||
&& imageObjectNames != null
|
||||
&& !imageObjectNames.isEmpty()
|
||||
&& !targetTokens.isEmpty()) {
|
||||
Object previous = targetTokens.getLast();
|
||||
Object previous = targetTokens.get(targetTokens.size() - 1);
|
||||
if (previous instanceof COSName cosName
|
||||
&& imageObjectNames.contains(cosName.getName())) {
|
||||
targetTokens.remove(targetTokens.size() - 1);
|
||||
@@ -5246,7 +5246,7 @@ public class PdfJsonConversionService {
|
||||
throws IOException {
|
||||
if (OperatorName.DRAW_OBJECT.equals(operator.getName())
|
||||
&& !operands.isEmpty()
|
||||
&& operands.getFirst() instanceof COSName name) {
|
||||
&& operands.get(0) instanceof COSName name) {
|
||||
currentXObjectName = name;
|
||||
}
|
||||
super.processOperator(operator, operands);
|
||||
|
||||
@@ -420,7 +420,7 @@ public class PdfJsonImageService {
|
||||
throws IOException {
|
||||
if (OperatorName.DRAW_OBJECT.equals(operator.getName())
|
||||
&& !operands.isEmpty()
|
||||
&& operands.getFirst() instanceof COSName name) {
|
||||
&& operands.get(0) instanceof COSName name) {
|
||||
currentXObjectName = name;
|
||||
}
|
||||
super.processOperator(operator, operands);
|
||||
|
||||
@@ -137,7 +137,7 @@ public class JobController {
|
||||
if (result.hasFiles() && !result.hasMultipleFiles()) {
|
||||
try {
|
||||
List<ResultFile> files = result.getAllResultFiles();
|
||||
ResultFile singleFile = files.getFirst();
|
||||
ResultFile singleFile = files.get(0);
|
||||
|
||||
byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId());
|
||||
return ResponseEntity.ok()
|
||||
|
||||
|
Before Width: | Height: | Size: 6.4 KiB After Width: | Height: | Size: 6.4 KiB |
@@ -1,18 +1 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 24 24">
|
||||
<symbol id="icon-redact-auto" viewBox="0 0 24 24"> <g id="Layer_2" data-name="Layer 2">
|
||||
<g id="Layer_1-2" data-name="Layer 1">
|
||||
<rect width="24" height="24" style="fill: none"/>
|
||||
<g>
|
||||
<path d="M17.541,15.64258a.91793.91793,0,0,1,.55469-.18555h1.1084a.91586.91586,0,0,1,.55469.18555,1.30889,1.30889,0,0,1,.40429.499,1.57206,1.57206,0,0,1,.15039.68457v5.47754H19.2041V20.21094H18.0957v2.09277H16.9873V16.82617a1.55843,1.55843,0,0,1,.15039-.68457A1.2979,1.2979,0,0,1,17.541,15.64258Zm1.66309,1.10547H18.0957v2.17187h1.1084Z" style="fill: currentColor"/>
|
||||
<path d="M5.68653,22.30351a2.00588,2.00588,0,0,1-2-2v-16A1.92585,1.92585,0,0,1,4.274,2.891a1.92585,1.92585,0,0,1,1.4125-.5875h8l6,6v5.66931h-2V9.30351h-5v-5h-7v16h9.74021v2Z" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="10.43189" width="4.33778" height="0.79501" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="12.16889" width="7.31192" height="1.21288" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="17.14555" width="7.31192" height="1.21288" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="14.32375" width="7.31192" height="0.57517" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="15.79848" width="5.25578" height="0.4475" style="fill: currentColor"/>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</symbol>
|
||||
</svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 24 24"><symbol id="icon-redact-auto" viewBox="0 0 24 24"><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><rect width="24" height="24" style="fill:none"/><g><path d="M17.541,15.64258a.91793.91793,0,0,1,.55469-.18555h1.1084a.91586.91586,0,0,1,.55469.18555,1.30889,1.30889,0,0,1,.40429.499,1.57206,1.57206,0,0,1,.15039.68457v5.47754H19.2041V20.21094H18.0957v2.09277H16.9873V16.82617a1.55843,1.55843,0,0,1,.15039-.68457A1.2979,1.2979,0,0,1,17.541,15.64258Zm1.66309,1.10547H18.0957v2.17187h1.1084Z" style="fill:currentColor"/><path d="M5.68653,22.30351a2.00588,2.00588,0,0,1-2-2v-16A1.92585,1.92585,0,0,1,4.274,2.891a1.92585,1.92585,0,0,1,1.4125-.5875h8l6,6v5.66931h-2V9.30351h-5v-5h-7v16h9.74021v2Z" style="fill:currentColor"/><rect width="4.338" height=".795" x="7.698" y="10.432" style="fill:currentColor"/><rect width="7.312" height="1.213" x="7.698" y="12.169" style="fill:currentColor"/><rect width="7.312" height="1.213" x="7.698" y="17.146" style="fill:currentColor"/><rect width="7.312" height=".575" x="7.698" y="14.324" style="fill:currentColor"/><rect width="5.256" height=".448" x="7.698" y="15.798" style="fill:currentColor"/></g></g></g></symbol></svg>
|
||||
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 1.3 KiB |
@@ -1,13 +1 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
||||
<svg width="800px" height="800px" viewBox="0 0 512 512" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<title>rename</title>
|
||||
<symbol id="icon-rename" viewBox="0 0 512 512">
|
||||
<g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
|
||||
<g id="Combined-Shape" fill="currentColor" transform="translate(42.666667, 64.000000)">
|
||||
<path d="M362.666667,1.42108547e-14 L362.666667,21.3333333 L320,21.333 L320,362.666 L362.666667,362.666667 L362.666667,384 L320,383.999 L320,384 L298.666667,384 L298.666,383.999 L256,384 L256,362.666667 L298.666,362.666 L298.666,21.333 L256,21.3333333 L256,1.42108547e-14 L362.666667,1.42108547e-14 Z M426.666667,64 L426.666667,320 L341.333333,320 L341.333333,277.333333 L384,277.333333 L384,106.666667 L341.333333,106.666667 L341.333333,64 L426.666667,64 Z M277.333333,64 L277.333333,320 L3.55271368e-14,320 L3.55271368e-14,64 L277.333333,64 Z M179.2,89.6 L149.333333,89.6 L149.333333,234.666667 C149.333333,248 148.5,256.333333 147.875,264.354167 L147.792993,265.422171 L147.792993,265.422171 L147.714003,266.48894 C147.417695,270.579012 147.2,274.696296 147.2,279.466667 L147.2,279.466667 L177.066667,279.466667 L177.066667,260.266667 C184.941497,273.926888 199.708077,282.130544 215.466667,281.6 C229.540046,281.805757 242.921593,275.508559 251.733333,264.533333 C263.162478,248.989677 269.832496,230.461848 270.933333,211.2 C270.933333,170.666667 249.6,142.933333 217.6,142.933333 C202.507405,142.999748 188.308689,150.099106 179.2,162.133333 L179.2,162.133333 L179.2,89.6 Z M119.466667,162.133333 C107.961824,149.843793 91.4322333,143.546807 74.6666667,145.066667 C57.6785115,144.485924 40.8138255,148.15216 25.6,155.733333 L25.6,155.733333 L34.1333333,177.066667 C45.3979052,171.147831 57.7246848,167.522308 70.4,166.4 C78.5613135,165.511423 86.6853595,168.371259 92.4903835,174.176283 C98.2954074,179.981307 101.155244,188.105353 100.266667,196.266667 L100.266667,196.266667 L100.266667,198.4 L78.9333333,198.4 C65.8181975,197.679203 52.705771,199.864608 40.5333333,204.8 C26.2806563,210.950309 17.6507691,225.621117 19.2,241.066667 C19.0625857,252.057651 23.6679763,262.574827 31.8381493,269.927982 C40.0083223,277.281138 50.9508304,280.757072 61.8666667,279.466667 C77.2795695,280.291768 92.2192911,274.001359 102.4,262.4 L102.4,262.4 L102.4,277.333333 L130.133333,277.333333 C128.292479,266.054406 127.577851,254.620365 128,243.2 L128,243.2 L128,204.8 C129.999138,190.023932 126.995128,175.003882 119.466667,162.133333 Z M98.1333333,213.333333 L98.1333333,238.933333 C92.082572,249.988391 80.836024,257.218314 68.2666667,258.133333 C63.0655139,258.520242 57.9538681,256.621996 54.2659359,252.934064 C50.5780036,249.246132 48.6797582,244.134486 49.0666667,238.933333 C49.0666667,224 59.7333333,215.466667 85.3333333,213.333333 L85.3333333,213.333333 L98.1333333,213.333333 Z M209.066667,166.4 C226.133333,166.4 238.933333,183.466667 238.933333,211.2 C238.933333,238.933333 228.266667,256 211.2,256 C197.298049,255.69869 184.825037,247.383349 179.2,234.666667 L179.2,234.666667 L179.2,187.733333 C185.154203,176.240507 196.263981,168.304951 209.066667,166.4 Z">
|
||||
</path>
|
||||
</g>
|
||||
</g>
|
||||
</symbol>
|
||||
</svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="800" height="800" version="1.1" viewBox="0 0 512 512"><title>rename</title><symbol id="icon-rename" viewBox="0 0 512 512"><g id="Page-1" fill="none" fill-rule="evenodd" stroke="none" stroke-width="1"><g id="Combined-Shape" fill="currentColor"><path d="M362.666667,1.42108547e-14 L362.666667,21.3333333 L320,21.333 L320,362.666 L362.666667,362.666667 L362.666667,384 L320,383.999 L320,384 L298.666667,384 L298.666,383.999 L256,384 L256,362.666667 L298.666,362.666 L298.666,21.333 L256,21.3333333 L256,1.42108547e-14 L362.666667,1.42108547e-14 Z M426.666667,64 L426.666667,320 L341.333333,320 L341.333333,277.333333 L384,277.333333 L384,106.666667 L341.333333,106.666667 L341.333333,64 L426.666667,64 Z M277.333333,64 L277.333333,320 L3.55271368e-14,320 L3.55271368e-14,64 L277.333333,64 Z M179.2,89.6 L149.333333,89.6 L149.333333,234.666667 C149.333333,248 148.5,256.333333 147.875,264.354167 L147.792993,265.422171 L147.792993,265.422171 L147.714003,266.48894 C147.417695,270.579012 147.2,274.696296 147.2,279.466667 L147.2,279.466667 L177.066667,279.466667 L177.066667,260.266667 C184.941497,273.926888 199.708077,282.130544 215.466667,281.6 C229.540046,281.805757 242.921593,275.508559 251.733333,264.533333 C263.162478,248.989677 269.832496,230.461848 270.933333,211.2 C270.933333,170.666667 249.6,142.933333 217.6,142.933333 C202.507405,142.999748 188.308689,150.099106 179.2,162.133333 L179.2,162.133333 L179.2,89.6 Z M119.466667,162.133333 C107.961824,149.843793 91.4322333,143.546807 74.6666667,145.066667 C57.6785115,144.485924 40.8138255,148.15216 25.6,155.733333 L25.6,155.733333 L34.1333333,177.066667 C45.3979052,171.147831 57.7246848,167.522308 70.4,166.4 C78.5613135,165.511423 86.6853595,168.371259 92.4903835,174.176283 C98.2954074,179.981307 101.155244,188.105353 100.266667,196.266667 L100.266667,196.266667 L100.266667,198.4 L78.9333333,198.4 C65.8181975,197.679203 52.705771,199.864608 40.5333333,204.8 C26.2806563,210.950309 17.6507691,225.621117 19.2,241.066667 C19.0625857,252.057651 23.6679763,262.574827 31.8381493,269.927982 C40.0083223,277.281138 50.9508304,280.757072 61.8666667,279.466667 C77.2795695,280.291768 92.2192911,274.001359 102.4,262.4 L102.4,262.4 L102.4,277.333333 L130.133333,277.333333 C128.292479,266.054406 127.577851,254.620365 128,243.2 L128,243.2 L128,204.8 C129.999138,190.023932 126.995128,175.003882 119.466667,162.133333 Z M98.1333333,213.333333 L98.1333333,238.933333 C92.082572,249.988391 80.836024,257.218314 68.2666667,258.133333 C63.0655139,258.520242 57.9538681,256.621996 54.2659359,252.934064 C50.5780036,249.246132 48.6797582,244.134486 49.0666667,238.933333 C49.0666667,224 59.7333333,215.466667 85.3333333,213.333333 L85.3333333,213.333333 L98.1333333,213.333333 Z M209.066667,166.4 C226.133333,166.4 238.933333,183.466667 238.933333,211.2 C238.933333,238.933333 228.266667,256 211.2,256 C197.298049,255.69869 184.825037,247.383349 179.2,234.666667 L179.2,234.666667 L179.2,187.733333 C185.154203,176.240507 196.263981,168.304951 209.066667,166.4 Z" transform="translate(42.666667, 64.000000)"/></g></g></symbol></svg>
|
||||
|
Before Width: | Height: | Size: 3.3 KiB After Width: | Height: | Size: 3.1 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 5.3 KiB |
@@ -4,6 +4,8 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
@@ -22,7 +24,10 @@ class MultipartConfigurationTest {
|
||||
void setUp() throws Exception {
|
||||
// Manually constructed config with a mocked service, so Spring env overrides do not apply.
|
||||
uploadLimitService = mock(UploadLimitService.class);
|
||||
configuration = new MultipartConfiguration(uploadLimitService);
|
||||
configuration = new MultipartConfiguration();
|
||||
Field field = MultipartConfiguration.class.getDeclaredField("uploadLimitService");
|
||||
field.setAccessible(true);
|
||||
field.set(configuration, uploadLimitService);
|
||||
}
|
||||
|
||||
@Nested
|
||||
|
||||
@@ -84,6 +84,12 @@ class MergeControllerGapTest {
|
||||
return (MultipartFile[]) m.invoke(null, files, fileOrder);
|
||||
}
|
||||
|
||||
private String[] parseClientFileIds(String value) throws Exception {
|
||||
Method m = MergeController.class.getDeclaredMethod("parseClientFileIds", String.class);
|
||||
m.setAccessible(true);
|
||||
return (String[]) m.invoke(mergeController, value);
|
||||
}
|
||||
|
||||
private long getPdfDateTimeSafe(MultipartFile file) throws Exception {
|
||||
Method m =
|
||||
MergeController.class.getDeclaredMethod("getPdfDateTimeSafe", MultipartFile.class);
|
||||
@@ -325,6 +331,58 @@ class MergeControllerGapTest {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- parseClientFileIds -------------------------------------------------
|
||||
|
||||
@Nested
|
||||
@DisplayName("parseClientFileIds")
|
||||
class ParseClientFileIds {
|
||||
|
||||
@Test
|
||||
@DisplayName("null input returns empty array")
|
||||
void nullReturnsEmpty() throws Exception {
|
||||
assertEquals(0, parseClientFileIds(null).length);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("blank input returns empty array")
|
||||
void blankReturnsEmpty() throws Exception {
|
||||
assertEquals(0, parseClientFileIds(" ").length);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("empty JSON array returns empty array")
|
||||
void emptyArrayReturnsEmpty() throws Exception {
|
||||
assertEquals(0, parseClientFileIds("[]").length);
|
||||
assertEquals(0, parseClientFileIds("[ ]").length);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("non-array text returns empty array")
|
||||
void nonArrayReturnsEmpty() throws Exception {
|
||||
assertEquals(0, parseClientFileIds("not-an-array").length);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("parses quoted, comma-separated ids and strips surrounding quotes")
|
||||
void parsesQuotedIds() throws Exception {
|
||||
String[] result = parseClientFileIds("[\"id1\", \"id2\",\"id3\"]");
|
||||
assertArrayEquals(new String[] {"id1", "id2", "id3"}, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("parses unquoted ids as-is after trimming")
|
||||
void parsesUnquotedIds() throws Exception {
|
||||
String[] result = parseClientFileIds("[a, b , c]");
|
||||
assertArrayEquals(new String[] {"a", "b", "c"}, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("single element array yields a one-element result")
|
||||
void singleElement() throws Exception {
|
||||
assertArrayEquals(new String[] {"only"}, parseClientFileIds("[\"only\"]"));
|
||||
}
|
||||
}
|
||||
|
||||
// ---- reorderFilesByProvidedOrder ----------------------------------------
|
||||
|
||||
@Nested
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.core.io.DefaultResourceLoader;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
|
||||
import stirling.software.SPDF.service.SharedSignatureService;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
class UIDataControllerTest {
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
@Test
|
||||
void getPipelineData_usesEachSourceFilenameWhenJsonContentIsIdentical() throws Exception {
|
||||
Path configDir = tempDir.resolve("defaultWebUIConfigs");
|
||||
Files.createDirectories(configDir);
|
||||
|
||||
String configJson = "{\"steps\":[]}";
|
||||
Files.writeString(configDir.resolve("first-config.json"), configJson);
|
||||
Files.writeString(configDir.resolve("second-config.json"), configJson);
|
||||
|
||||
ApplicationProperties applicationProperties = mock(ApplicationProperties.class);
|
||||
SharedSignatureService signatureService = mock(SharedSignatureService.class);
|
||||
RuntimePathConfig runtimePathConfig = mock(RuntimePathConfig.class);
|
||||
|
||||
when(runtimePathConfig.getPipelineDefaultWebUiConfigs()).thenReturn(configDir.toString());
|
||||
|
||||
UIDataController controller =
|
||||
new UIDataController(
|
||||
applicationProperties,
|
||||
signatureService,
|
||||
null,
|
||||
new DefaultResourceLoader(),
|
||||
runtimePathConfig,
|
||||
new ObjectMapper());
|
||||
|
||||
ResponseEntity<UIDataController.PipelineData> response = controller.getPipelineData();
|
||||
|
||||
assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
|
||||
UIDataController.PipelineData body = response.getBody();
|
||||
assertThat(body).isNotNull();
|
||||
|
||||
List<Map<String, String>> configsWithNames = body.getPipelineConfigsWithNames();
|
||||
assertThat(configsWithNames).hasSize(2);
|
||||
assertThat(configsWithNames)
|
||||
.extracting(entry -> entry.get("name"))
|
||||
.containsExactlyInAnyOrder("first-config", "second-config");
|
||||
assertThat(configsWithNames)
|
||||
.extracting(entry -> entry.get("json"))
|
||||
.containsOnly(configJson);
|
||||
}
|
||||
}
|
||||
@@ -204,7 +204,7 @@ class ConvertOfficeControllerTest {
|
||||
inv -> {
|
||||
// unoconvert writes directly to the output path (last arg)
|
||||
List<String> command = inv.getArgument(0);
|
||||
Path out = Path.of(command.getLast());
|
||||
Path out = Path.of(command.get(command.size() - 1));
|
||||
Files.writeString(out, "%PDF-1.4 produced");
|
||||
return result;
|
||||
});
|
||||
@@ -239,7 +239,7 @@ class ConvertOfficeControllerTest {
|
||||
inv -> {
|
||||
// soffice writes <basename>.pdf into the --outdir (workDir)
|
||||
List<String> command = inv.getArgument(0);
|
||||
Path inputPath = Path.of(command.getLast());
|
||||
Path inputPath = Path.of(command.get(command.size() - 1));
|
||||
Path out = inputPath.getParent().resolve("report.pdf");
|
||||
Files.writeString(out, "%PDF soffice");
|
||||
return result;
|
||||
@@ -311,7 +311,7 @@ class ConvertOfficeControllerTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> command = inv.getArgument(0);
|
||||
Path inputPath = Path.of(command.getLast());
|
||||
Path inputPath = Path.of(command.get(command.size() - 1));
|
||||
Path out = inputPath.getParent().resolve("report.pdf");
|
||||
Files.write(out, new byte[0]);
|
||||
return result;
|
||||
@@ -344,7 +344,7 @@ class ConvertOfficeControllerTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> command = inv.getArgument(0);
|
||||
Path inputPath = Path.of(command.getLast());
|
||||
Path inputPath = Path.of(command.get(command.size() - 1));
|
||||
Path out = inputPath.getParent().resolve("page.pdf");
|
||||
Files.writeString(out, "%PDF html");
|
||||
return result;
|
||||
@@ -398,7 +398,7 @@ class ConvertOfficeControllerTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> command = inv.getArgument(0);
|
||||
Path inputPath = Path.of(command.getLast());
|
||||
Path inputPath = Path.of(command.get(command.size() - 1));
|
||||
Path out = inputPath.getParent().resolve("report.pdf");
|
||||
Files.writeString(out, "%PDF produced");
|
||||
return result;
|
||||
|
||||
@@ -197,7 +197,7 @@ class ConvertPDFToPDFAMoreTest {
|
||||
// qpdf normalize/clean writes its (last-arg) output file
|
||||
if (command.contains("--normalize-content=y")) {
|
||||
// qpdf produced file is the last argument
|
||||
Path out = Path.of(command.getLast());
|
||||
Path out = Path.of(command.get(command.size() - 1));
|
||||
Files.write(out, simplePdfBytes());
|
||||
}
|
||||
return okResult;
|
||||
|
||||
@@ -262,7 +262,7 @@ class ConvertPdfToVideoControllerTest {
|
||||
assertTrue(command.contains("+faststart"));
|
||||
assertFalse(command.contains("libvpx-vp9"));
|
||||
// Output path is always the last argument.
|
||||
assertEquals(backing.getAbsolutePath(), command.getLast());
|
||||
assertEquals(backing.getAbsolutePath(), command.get(command.size() - 1));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -278,7 +278,7 @@ class ConvertPdfToVideoControllerTest {
|
||||
assertTrue(command.contains("30"));
|
||||
assertFalse(command.contains("libx264"));
|
||||
assertFalse(command.contains("+faststart"));
|
||||
assertEquals(backing.getAbsolutePath(), command.getLast());
|
||||
assertEquals(backing.getAbsolutePath(), command.get(command.size() - 1));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -195,7 +195,7 @@ class CompressControllerMoreTest {
|
||||
|
||||
// The qpdf output path is the last argument of the command.
|
||||
private static Path qpdfOutputPath(List<String> command) {
|
||||
return Path.of(command.getLast());
|
||||
return Path.of(command.get(command.size() - 1));
|
||||
}
|
||||
|
||||
/** Stub gs to write a valid PDF to its output file and report success. */
|
||||
|
||||
@@ -162,7 +162,7 @@ class RemoveImagesControllerTest {
|
||||
/** Counts every PDImageXObject reachable through page + nested form resources. */
|
||||
private int countImagesInSavedOutput() throws IOException {
|
||||
assertFalse(savedTempFiles.isEmpty(), "expected the controller to create a temp file");
|
||||
File out = savedTempFiles.getLast();
|
||||
File out = savedTempFiles.get(savedTempFiles.size() - 1);
|
||||
try (PDDocument doc = Loader.loadPDF(out)) {
|
||||
int count = 0;
|
||||
for (PDPage page : doc.getPages()) {
|
||||
@@ -245,7 +245,7 @@ class RemoveImagesControllerTest {
|
||||
|
||||
assertEquals(0, countImagesInSavedOutput());
|
||||
// page count must be preserved
|
||||
File out = savedTempFiles.getLast();
|
||||
File out = savedTempFiles.get(savedTempFiles.size() - 1);
|
||||
try (PDDocument result = Loader.loadPDF(out)) {
|
||||
assertEquals(3, result.getNumberOfPages());
|
||||
}
|
||||
|
||||
@@ -98,8 +98,12 @@ class RepairControllerMoreTest {
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a valid PDF to the given output path, mimicking a successful tool run. */
|
||||
private static void writeValidPdfTo(Path out) throws Exception {
|
||||
/**
|
||||
* Writes a valid PDF to the path at the given command index, mimicking a successful tool run.
|
||||
*/
|
||||
private static void writeValidPdfTo(List<String> command, int outputPathIndex)
|
||||
throws Exception {
|
||||
Path out = Path.of(command.get(outputPathIndex));
|
||||
byte[] pdf = buildPdfBytes(1);
|
||||
Files.write(out, pdf);
|
||||
}
|
||||
@@ -129,7 +133,7 @@ class RepairControllerMoreTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> cmd = inv.getArgument(0);
|
||||
writeValidPdfTo(Path.of(cmd.get(2)));
|
||||
writeValidPdfTo(cmd, 2);
|
||||
return okResult;
|
||||
});
|
||||
|
||||
@@ -172,7 +176,7 @@ class RepairControllerMoreTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> cmd = inv.getArgument(0);
|
||||
writeValidPdfTo(Path.of(cmd.getLast()));
|
||||
writeValidPdfTo(cmd, cmd.size() - 1);
|
||||
return okResult;
|
||||
});
|
||||
|
||||
@@ -212,7 +216,7 @@ class RepairControllerMoreTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> cmd = inv.getArgument(0);
|
||||
writeValidPdfTo(Path.of(cmd.getLast()));
|
||||
writeValidPdfTo(cmd, cmd.size() - 1);
|
||||
return okResult;
|
||||
});
|
||||
|
||||
@@ -252,7 +256,7 @@ class RepairControllerMoreTest {
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
List<String> cmd = inv.getArgument(0);
|
||||
writeValidPdfTo(Path.of(cmd.getLast()));
|
||||
writeValidPdfTo(cmd, cmd.size() - 1);
|
||||
return okResult;
|
||||
});
|
||||
|
||||
|
||||
@@ -299,6 +299,18 @@ class RedactControllerMoreTest {
|
||||
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("null listOfText throws an illegal-argument error before any load")
|
||||
void nullPatternsThrows() throws Exception {
|
||||
RedactPdfRequest request = new RedactPdfRequest();
|
||||
request.setFileInput(pdfFile(new byte[] {1, 2, 3}));
|
||||
request.setListOfText(null);
|
||||
|
||||
assertThatThrownBy(() -> controller.redactPdf(request))
|
||||
.isInstanceOf(RuntimeException.class);
|
||||
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("null file input is reported as a failure")
|
||||
void nullFileThrows() {
|
||||
|
||||
@@ -49,7 +49,15 @@ class UploadLimitServiceTest {
|
||||
systemProps = mock(ApplicationProperties.System.class);
|
||||
when(applicationProperties.getSystem()).thenReturn(systemProps);
|
||||
|
||||
uploadLimitService = new UploadLimitService(applicationProperties);
|
||||
uploadLimitService = new UploadLimitService();
|
||||
// inject mock
|
||||
try {
|
||||
var field = UploadLimitService.class.getDeclaredField("applicationProperties");
|
||||
field.setAccessible(true);
|
||||
field.set(uploadLimitService, applicationProperties);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "getReadableUploadLimit case #{index}: rawValue={0}, expected={1}")
|
||||
|
||||
@@ -142,6 +142,9 @@ public class AuditAspect {
|
||||
auditService.addTimingData(
|
||||
auditData, startTime, resp, auditedAnnotation.level(), isHttpRequest);
|
||||
|
||||
// Merge controller-set policy context + the internal-automation marker onto the event.
|
||||
auditService.addAutomationContext(auditData, req);
|
||||
|
||||
// Resolve the event type based on annotation and context
|
||||
String httpMethod = null;
|
||||
String path = null;
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.proprietary.audit;
|
||||
|
||||
/**
|
||||
* Request-scoped keys a controller can set to enrich its own audit event with context the generic
|
||||
* aspect can't infer from the HTTP request alone (e.g. the policy a pipeline run belongs to). The
|
||||
* aspect copies these into the audit data in its {@code finally} block, after the controller body
|
||||
* has run. See {@code AuditService#addAutomationContext}.
|
||||
*/
|
||||
public final class AuditContext {
|
||||
|
||||
/** Request attribute: the name of the policy/pipeline a run executes. */
|
||||
public static final String REQ_ATTR_POLICY_NAME = "stirling.audit.policyName";
|
||||
|
||||
/** Request attribute: the ordered tool endpoint paths a run executes. */
|
||||
public static final String REQ_ATTR_POLICY_STEPS = "stirling.audit.policySteps";
|
||||
|
||||
private AuditContext() {}
|
||||
}
|
||||
@@ -205,6 +205,10 @@ public class ControllerAuditAspect {
|
||||
// Call auditService but with isHttpRequest=true to skip additional timing
|
||||
auditService.addTimingData(data, start, resp, level, true);
|
||||
|
||||
// Merge controller-set policy context + the internal-automation marker (set after
|
||||
// the body ran, so it must happen here rather than with the pre-proceed HTTP data).
|
||||
auditService.addAutomationContext(data, req);
|
||||
|
||||
// Resolve the event type using the unified method
|
||||
AuditEventType eventType =
|
||||
auditService.resolveEventType(
|
||||
|
||||
@@ -133,29 +133,45 @@ public final class S3Clients {
|
||||
* storage.s3.allow-private-endpoints=true}.
|
||||
*/
|
||||
static void validateEndpointHost(URI endpoint, boolean allowPrivate) {
|
||||
validateEndpointHost(
|
||||
endpoint,
|
||||
allowPrivate,
|
||||
"storage.s3.endpoint",
|
||||
"set storage.s3.allow-private-endpoints=true to opt in"
|
||||
+ " (e.g. for MinIO or in-cluster S3).");
|
||||
}
|
||||
|
||||
/**
|
||||
* The same private-address guard for S3 endpoints configured outside the {@code storage.s3.*}
|
||||
* block (e.g. per-source policy config), with the setting named in messages supplied by the
|
||||
* caller.
|
||||
*/
|
||||
public static void validateEndpointHost(
|
||||
URI endpoint, boolean allowPrivate, String settingName, String optInHint) {
|
||||
if (allowPrivate) {
|
||||
return;
|
||||
}
|
||||
String host = endpoint.getHost();
|
||||
if (host == null || host.isBlank()) {
|
||||
throw new IllegalStateException("storage.s3.endpoint must include a host: " + endpoint);
|
||||
throw new IllegalStateException(settingName + " must include a host: " + endpoint);
|
||||
}
|
||||
InetAddress[] addresses;
|
||||
try {
|
||||
addresses = InetAddress.getAllByName(host);
|
||||
} catch (UnknownHostException e) {
|
||||
throw new IllegalStateException(
|
||||
"Unable to resolve storage.s3.endpoint host '" + host + "'", e);
|
||||
"Unable to resolve " + settingName + " host '" + host + "'", e);
|
||||
}
|
||||
for (InetAddress address : addresses) {
|
||||
if (isPrivateOrLocal(address)) {
|
||||
throw new IllegalStateException(
|
||||
"storage.s3.endpoint host '"
|
||||
settingName
|
||||
+ " host '"
|
||||
+ host
|
||||
+ "' resolves to private/link-local address "
|
||||
+ address.getHostAddress()
|
||||
+ "; set storage.s3.allow-private-endpoints=true to opt in"
|
||||
+ " (e.g. for MinIO or in-cluster S3).");
|
||||
+ "; "
|
||||
+ optInHint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,10 +54,21 @@ public class CustomAuditEventRepository implements AuditEventRepository {
|
||||
return;
|
||||
}
|
||||
String rid = MDC.get("requestId");
|
||||
String apiKeyLabel =
|
||||
MDC.get(
|
||||
stirling.software.proprietary.security.service
|
||||
.ApiKeyAuthenticationService.AUDIT_LABEL_MDC_KEY);
|
||||
|
||||
if (rid != null) {
|
||||
if (rid != null || apiKeyLabel != null) {
|
||||
clean = new java.util.HashMap<>(clean);
|
||||
clean.put("requestId", rid);
|
||||
if (rid != null) {
|
||||
clean.put("requestId", rid);
|
||||
}
|
||||
// Named key that made the request; surfaces as the doc source in the processor
|
||||
// feed.
|
||||
if (apiKeyLabel != null) {
|
||||
clean.put("__apiKeyLabel", apiKeyLabel);
|
||||
}
|
||||
}
|
||||
|
||||
String source = MDC.get("auditSource");
|
||||
|
||||
@@ -143,12 +143,19 @@ public class AuditRestController {
|
||||
@RequestParam(value = "period", defaultValue = "week") String period) {
|
||||
|
||||
// Calculate days based on period
|
||||
int days =
|
||||
switch (period.toLowerCase()) {
|
||||
case "day" -> 1;
|
||||
case "month" -> 30;
|
||||
default -> 7;
|
||||
};
|
||||
int days;
|
||||
switch (period.toLowerCase()) {
|
||||
case "day":
|
||||
days = 1;
|
||||
break;
|
||||
case "month":
|
||||
days = 30;
|
||||
break;
|
||||
case "week":
|
||||
default:
|
||||
days = 7;
|
||||
break;
|
||||
}
|
||||
|
||||
// Get events from the specified period
|
||||
Instant startDate = Instant.now().minus(java.time.Duration.ofDays(days));
|
||||
@@ -262,12 +269,19 @@ public class AuditRestController {
|
||||
@RequestParam(value = "period", defaultValue = "week") String period) {
|
||||
|
||||
// Calculate days based on period
|
||||
int days =
|
||||
switch (period.toLowerCase()) {
|
||||
case "day" -> 1;
|
||||
case "month" -> 30;
|
||||
default -> 7;
|
||||
};
|
||||
int days;
|
||||
switch (period.toLowerCase()) {
|
||||
case "day":
|
||||
days = 1;
|
||||
break;
|
||||
case "month":
|
||||
days = 30;
|
||||
break;
|
||||
case "week":
|
||||
default:
|
||||
days = 7;
|
||||
break;
|
||||
}
|
||||
|
||||
// Get events from the specified period and previous period
|
||||
Instant now = Instant.now();
|
||||
@@ -740,7 +754,7 @@ public class AuditRestController {
|
||||
List<Map<String, Object>> files =
|
||||
(List<Map<String, Object>>) eventData.get("files");
|
||||
if (files != null && !files.isEmpty()) {
|
||||
Map<String, Object> firstFile = files.getFirst();
|
||||
Map<String, Object> firstFile = files.get(0);
|
||||
data.put("documentname", String.valueOf(firstFile.getOrDefault("name", "")));
|
||||
data.put("author", String.valueOf(firstFile.getOrDefault("pdfAuthor", "")));
|
||||
data.put("filehash", String.valueOf(firstFile.getOrDefault("fileHash", "")));
|
||||
|
||||
@@ -4,6 +4,7 @@ import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
@@ -38,6 +39,9 @@ import stirling.software.proprietary.security.database.repository.UserRepository
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@RequiredArgsConstructor
|
||||
@EnterpriseEndpoint
|
||||
// Self-hosted only: counts are server-wide. On SaaS this endpoint is owned by the team-scoped
|
||||
// SaasFleetUsageController (@Profile("saas")) so one backend can't leak another tenant's usage.
|
||||
@Profile("!saas")
|
||||
public class FleetUsageController {
|
||||
|
||||
private final PersistentAuditEventRepository auditRepository;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package stirling.software.proprietary.controller.api;
|
||||
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.annotations.api.ProprietaryUiDataApi;
|
||||
import stirling.software.proprietary.model.api.apikey.CreateApiKeyRequest;
|
||||
import stirling.software.proprietary.model.api.apikey.CreatedApiKeyDto;
|
||||
import stirling.software.proprietary.model.api.apikey.PortalApiKeysResponse;
|
||||
import stirling.software.proprietary.security.service.ApiKeyManagementService;
|
||||
|
||||
/**
|
||||
* Real backing for the portal Infrastructure → API Keys tab: list/create/revoke named, personal API
|
||||
* keys. Replaces the former portal-only mock endpoint. Not gated behind an Enterprise license - API
|
||||
* keys are a core auth feature available on every self-hosted instance.
|
||||
*/
|
||||
@ProprietaryUiDataApi
|
||||
@RequiredArgsConstructor
|
||||
public class PortalApiKeysController {
|
||||
|
||||
private final ApiKeyManagementService apiKeyManagementService;
|
||||
|
||||
// tier accepted for endpoint symmetry with the other infra tabs; ignored here.
|
||||
@GetMapping("/infrastructure/api-keys")
|
||||
@Operation(summary = "List API keys", description = "The caller's personal API keys.")
|
||||
public ResponseEntity<PortalApiKeysResponse> list(
|
||||
@RequestParam(value = "tier", required = false) String tier) {
|
||||
return ResponseEntity.ok(apiKeyManagementService.listVisibleKeys());
|
||||
}
|
||||
|
||||
@PostMapping("/infrastructure/api-keys")
|
||||
@Operation(
|
||||
summary = "Create an API key",
|
||||
description = "Mints a personal key and returns its one-time secret.")
|
||||
public ResponseEntity<CreatedApiKeyDto> create(@RequestBody CreateApiKeyRequest request) {
|
||||
return ResponseEntity.ok(apiKeyManagementService.createKey(request));
|
||||
}
|
||||
|
||||
@DeleteMapping("/infrastructure/api-keys/{id}")
|
||||
@Operation(summary = "Revoke an API key", description = "Disables a key the caller owns.")
|
||||
public ResponseEntity<Void> revoke(@PathVariable("id") Long id) {
|
||||
apiKeyManagementService.revokeKey(id);
|
||||
return ResponseEntity.noContent().build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package stirling.software.proprietary.integration.crypto;
|
||||
|
||||
import jakarta.persistence.AttributeConverter;
|
||||
import jakarta.persistence.Converter;
|
||||
|
||||
/**
|
||||
* {@link EncryptedStringConverter} for columns that held plaintext before encryption shipped:
|
||||
* writes are always encrypted, but a stored value that is not valid ciphertext is returned as-is,
|
||||
* so pre-encryption rows keep loading and become encrypted on their next save. The discrimination
|
||||
* is exact for JSON payloads, which can never be mistaken for ciphertext ('{' is not in the Base64
|
||||
* alphabet). The trade-off is that a genuinely corrupted ciphertext surfaces as garbage to the
|
||||
* caller's parser instead of failing here.
|
||||
*/
|
||||
@Converter
|
||||
public class LenientEncryptedStringConverter implements AttributeConverter<String, String> {
|
||||
|
||||
@Override
|
||||
public String convertToDatabaseColumn(String attribute) {
|
||||
return CredentialEncryption.encrypt(attribute);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String convertToEntityAttribute(String dbData) {
|
||||
try {
|
||||
return CredentialEncryption.decrypt(dbData);
|
||||
} catch (IllegalArgumentException | IllegalStateException e) {
|
||||
// Not ciphertext: legacy plaintext from before encryption shipped.
|
||||
return dbData;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,10 @@ public class IntegrationConfigService {
|
||||
private final OwnershipService ownership;
|
||||
private final SecretMasker secretMasker;
|
||||
private final ResourceGrantRepository grantRepository;
|
||||
// Bean-discovered extension points: features that understand a type contribute its config
|
||||
// schema and report what still references a config, without this module depending on them.
|
||||
private final List<IntegrationConfigValidator> validators;
|
||||
private final List<IntegrationConfigUsageCheck> usageChecks;
|
||||
|
||||
// ---- commands ----
|
||||
|
||||
@@ -66,13 +70,21 @@ public class IntegrationConfigService {
|
||||
? DefaultAccessPolicy.EXPLICIT_ONLY
|
||||
: request.defaultAccess());
|
||||
|
||||
// TEAM scope may omit the team id: default to the caller's own team so clients (the
|
||||
// portal) need not know it. assignOwnership still enforces admin-or-leader of that team.
|
||||
Long ownerTeamId = request.ownerTeamId();
|
||||
if (ownerTeamId == null && scope == OwnerScope.TEAM && currentUser.getTeam() != null) {
|
||||
ownerTeamId = currentUser.getTeam().getId();
|
||||
}
|
||||
ownership.assignOwnership(
|
||||
cfg,
|
||||
scope,
|
||||
request.ownerTeamId(),
|
||||
ownerTeamId,
|
||||
currentUser,
|
||||
() -> lockedServerExists(cfg.getIntegrationType()));
|
||||
cfg.setConfig(writeJson(secretMasker.sanitize(request.config())));
|
||||
Map<String, Object> config = secretMasker.sanitize(request.config());
|
||||
validateConfig(cfg.getIntegrationType(), config);
|
||||
cfg.setConfig(writeJson(config));
|
||||
return repository.save(cfg);
|
||||
}
|
||||
|
||||
@@ -101,8 +113,10 @@ public class IntegrationConfigService {
|
||||
cfg.setDefaultAccess(request.defaultAccess());
|
||||
}
|
||||
if (request.config() != null) {
|
||||
cfg.setConfig(
|
||||
writeJson(secretMasker.merge(readJson(cfg.getConfig()), request.config())));
|
||||
Map<String, Object> merged =
|
||||
secretMasker.merge(readJson(cfg.getConfig()), request.config());
|
||||
validateConfig(cfg.getIntegrationType(), merged);
|
||||
cfg.setConfig(writeJson(merged));
|
||||
}
|
||||
return repository.save(cfg);
|
||||
}
|
||||
@@ -113,6 +127,15 @@ public class IntegrationConfigService {
|
||||
if (!ownership.canManage(TYPE, cfg, currentUser)) {
|
||||
throw forbidden("You cannot manage this integration");
|
||||
}
|
||||
// Refuse to pull a connection out from under whatever still references it.
|
||||
List<String> usages =
|
||||
usageChecks.stream()
|
||||
.flatMap(check -> check.usagesOf(cfg.getId()).stream())
|
||||
.toList();
|
||||
if (!usages.isEmpty()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.CONFLICT, "Integration is in use by: " + String.join(", ", usages));
|
||||
}
|
||||
// Drop grants sharing this config so they do not dangle as dead rows.
|
||||
grantRepository.deleteByResourceTypeAndResourceId(TYPE, String.valueOf(cfg.getId()));
|
||||
repository.delete(cfg);
|
||||
@@ -188,6 +211,19 @@ public class IntegrationConfigService {
|
||||
|
||||
// ---- integration-specific glue ----
|
||||
|
||||
/** Runs every registered validator for the type; unknown types save free-form. */
|
||||
private void validateConfig(IntegrationType type, Map<String, Object> config) {
|
||||
for (IntegrationConfigValidator validator : validators) {
|
||||
if (validator.type() == type) {
|
||||
try {
|
||||
validator.validate(config == null ? Map.of() : config);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A non-admin can't create a personal config of a type an admin has locked at server scope. */
|
||||
private boolean lockedServerExists(IntegrationType type) {
|
||||
return repository.findByScope(OwnerScope.SERVER).stream()
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package stirling.software.proprietary.integration.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Reports what still references an integration config, so deletion can be refused instead of
|
||||
* pulling a connection out from under a live consumer. Implementations are beans discovered by
|
||||
* {@link IntegrationConfigService} (e.g. the policy subsystem reporting sources and pipelines that
|
||||
* reference a connection).
|
||||
*/
|
||||
public interface IntegrationConfigUsageCheck {
|
||||
|
||||
/** Human-readable labels of everything still using the config; empty when unreferenced. */
|
||||
List<String> usagesOf(long configId);
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package stirling.software.proprietary.integration.service;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
|
||||
/**
|
||||
* Validates one integration type's config map at save time. Implementations are beans discovered by
|
||||
* {@link IntegrationConfigService}, so the feature that understands a type (e.g. the policy S3
|
||||
* backend) owns its schema without the integration module depending on it. Types with no registered
|
||||
* validator save free-form.
|
||||
*/
|
||||
public interface IntegrationConfigValidator {
|
||||
|
||||
/** The type this validator understands. */
|
||||
IntegrationType type();
|
||||
|
||||
/**
|
||||
* Validates the config as it will be stored (secrets already sanitized/merged, so values are
|
||||
* real, never the redaction mask). Throws {@link IllegalArgumentException} on bad config.
|
||||
*/
|
||||
void validate(Map<String, Object> config);
|
||||
}
|
||||
@@ -24,8 +24,8 @@ import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/**
|
||||
* API-key auth for the MCP endpoint: validates a Stirling per-user API key and binds the request to
|
||||
* that user with the MCP scopes.
|
||||
* API-key auth for the MCP endpoint: validates a Stirling API key and binds the request to that
|
||||
* user with the MCP scopes.
|
||||
*/
|
||||
@Slf4j
|
||||
public class McpApiKeyAuthFilter extends OncePerRequestFilter {
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
package stirling.software.proprietary.model.api.apikey;
|
||||
|
||||
/** Create-key request body from the portal: just a display name for the new personal key. */
|
||||
public record CreateApiKeyRequest(String name) {}
|
||||
@@ -0,0 +1,7 @@
|
||||
package stirling.software.proprietary.model.api.apikey;
|
||||
|
||||
import lombok.Builder;
|
||||
|
||||
/** Returned once when a key is created: the row plus the plaintext secret, never persisted. */
|
||||
@Builder
|
||||
public record CreatedApiKeyDto(PortalApiKeyDto key, String secret) {}
|
||||
@@ -0,0 +1,21 @@
|
||||
package stirling.software.proprietary.model.api.apikey;
|
||||
|
||||
import lombok.Builder;
|
||||
|
||||
/**
|
||||
* One API key as shown in the portal Infrastructure → API Keys tab. Never carries the secret; that
|
||||
* is returned once from {@link CreatedApiKeyDto} at creation time.
|
||||
*/
|
||||
@Builder
|
||||
public record PortalApiKeyDto(
|
||||
String id,
|
||||
String name,
|
||||
String prefix,
|
||||
String created,
|
||||
String lastUsed,
|
||||
/** "active" | "revoked". */
|
||||
String status,
|
||||
long usageToday,
|
||||
long usageMonth,
|
||||
/** Lifetime request count for the key. */
|
||||
long usageTotal) {}
|
||||
@@ -0,0 +1,9 @@
|
||||
package stirling.software.proprietary.model.api.apikey;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Builder;
|
||||
|
||||
/** Payload for the API Keys tab: the personal keys the caller owns. */
|
||||
@Builder
|
||||
public record PortalApiKeysResponse(List<PortalApiKeyDto> keys) {}
|
||||
@@ -17,6 +17,9 @@ public class InfraAuditSummary {
|
||||
@Schema(description = "Total events in the returned window", example = "40")
|
||||
private int totalEvents;
|
||||
|
||||
@Schema(description = "Policy-run events", example = "3")
|
||||
private int policy;
|
||||
|
||||
@Schema(description = "Processing-category events", example = "24")
|
||||
private int processing;
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestPart;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.context.request.RequestContextHolder;
|
||||
import org.springframework.web.context.request.ServletRequestAttributes;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
@@ -32,6 +34,7 @@ import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.validation.Valid;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -42,14 +45,18 @@ import stirling.software.common.model.job.JobResponse;
|
||||
import stirling.software.common.service.JobOwnershipService;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.proprietary.audit.AuditContext;
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.PolicyValidator;
|
||||
import stirling.software.proprietary.policy.engine.SweepOutcome;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
@@ -58,12 +65,15 @@ import stirling.software.proprietary.policy.model.PolicyRunView;
|
||||
import stirling.software.proprietary.policy.overview.PoliciesOverviewResponse;
|
||||
import stirling.software.proprietary.policy.overview.PolicyOverviewService;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
import stirling.software.proprietary.policy.source.EditorSource;
|
||||
import stirling.software.proprietary.policy.source.SourceAccessGuard;
|
||||
import stirling.software.proprietary.policy.source.SourceDocCounter;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
|
||||
import stirling.software.proprietary.policy.trigger.TriggerInfo;
|
||||
import stirling.software.proprietary.util.SecretMasker;
|
||||
|
||||
/**
|
||||
* Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
|
||||
@@ -83,6 +93,7 @@ public class PolicyController {
|
||||
private final PolicyStore policyStore;
|
||||
private final SourceStore sourceStore;
|
||||
private final SourceAccessGuard sourceAccessGuard;
|
||||
private final SourceDocCounter docCounter;
|
||||
private final PolicyValidator policyValidator;
|
||||
private final PolicyAccessGuard policyAccessGuard;
|
||||
private final PolicyManagementAuthority policyManagementAuthority;
|
||||
@@ -107,11 +118,14 @@ public class PolicyController {
|
||||
@RequestPart("json") PipelineDefinition definition,
|
||||
@Valid @ModelAttribute PolicyRunFiles files)
|
||||
throws IOException {
|
||||
stampPolicyAudit(definition);
|
||||
requireRunnable(definition);
|
||||
validateAdHocOutput(definition);
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
String runId =
|
||||
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId();
|
||||
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
|
||||
PolicyRunHandle handle =
|
||||
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP);
|
||||
recordEditorDocs(inputs);
|
||||
return ResponseEntity.accepted().body(new JobResponse<>(true, handle.runId(), null));
|
||||
}
|
||||
|
||||
@PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@@ -125,7 +139,9 @@ public class PolicyController {
|
||||
@RequestPart("json") PipelineDefinition definition,
|
||||
@Valid @ModelAttribute PolicyRunFiles files)
|
||||
throws IOException {
|
||||
stampPolicyAudit(definition);
|
||||
requireRunnable(definition);
|
||||
validateAdHocOutput(definition);
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
|
||||
SseEmitter emitter =
|
||||
@@ -133,6 +149,7 @@ public class PolicyController {
|
||||
emitter.onError(e -> log.warn("Policy run SSE emitter error", e));
|
||||
|
||||
PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter));
|
||||
recordEditorDocs(inputs);
|
||||
// whenComplete runs on the worker thread after the run finishes, so the terminal event
|
||||
// never races the step events.
|
||||
handle.completion()
|
||||
@@ -202,7 +219,7 @@ public class PolicyController {
|
||||
+ " assigned; returns the stored policy with its id.")
|
||||
public ResponseEntity<Policy> savePolicy(@RequestBody Policy policy) {
|
||||
requirePolicyEditingAllowed();
|
||||
Policy owned = resolveOwnership(policy);
|
||||
Policy owned = withStoredOutputSecrets(resolveOwnership(policy));
|
||||
requireAccessibleSources(owned);
|
||||
try {
|
||||
policyValidator.validate(owned);
|
||||
@@ -213,7 +230,7 @@ public class PolicyController {
|
||||
// Re-sync trigger registrations now so a new/changed folder-watch policy starts being
|
||||
// watched immediately instead of after the next reconcile sweep.
|
||||
policyTriggerManager.notifyPoliciesChanged();
|
||||
return ResponseEntity.ok(saved);
|
||||
return ResponseEntity.ok(withMaskedOutputSecrets(saved));
|
||||
}
|
||||
|
||||
@PutMapping("/order")
|
||||
@@ -282,6 +299,50 @@ public class PolicyController {
|
||||
teamId);
|
||||
}
|
||||
|
||||
/** Output secrets never leave the server: reads return the redaction sentinel instead. */
|
||||
private static Policy withMaskedOutputSecrets(Policy policy) {
|
||||
return withOutput(
|
||||
policy,
|
||||
new OutputSpec(
|
||||
policy.output().type(), SecretMasker.mask(policy.output().options())));
|
||||
}
|
||||
|
||||
/**
|
||||
* An edit that round-trips a masked read sends output secrets back as the sentinel; restore
|
||||
* them from the stored policy so saving without re-typing keeps them (validation then runs
|
||||
* against the real values).
|
||||
*/
|
||||
private Policy withStoredOutputSecrets(Policy incoming) {
|
||||
if (incoming.id() == null || incoming.id().isBlank()) {
|
||||
return incoming;
|
||||
}
|
||||
return policyStore
|
||||
.get(incoming.id())
|
||||
.map(
|
||||
existing ->
|
||||
withOutput(
|
||||
incoming,
|
||||
new OutputSpec(
|
||||
incoming.output().type(),
|
||||
SecretMasker.restoreRedacted(
|
||||
incoming.output().options(),
|
||||
existing.output().options()))))
|
||||
.orElse(incoming);
|
||||
}
|
||||
|
||||
private static Policy withOutput(Policy policy, OutputSpec output) {
|
||||
return new Policy(
|
||||
policy.id(),
|
||||
policy.name(),
|
||||
policy.owner(),
|
||||
policy.enabled(),
|
||||
policy.trigger(),
|
||||
policy.sourceIds(),
|
||||
policy.steps(),
|
||||
output,
|
||||
policy.teamId());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creating, editing, pausing/resuming, and deleting policies requires the editor role for the
|
||||
* caller's team — a team leader on SaaS (see {@link PolicyManagementAuthority}); the global
|
||||
@@ -306,9 +367,14 @@ public class PolicyController {
|
||||
@GetMapping
|
||||
@Operation(
|
||||
summary = "List policies",
|
||||
description = "Lists the policies belonging to the caller's team.")
|
||||
description =
|
||||
"Lists the policies belonging to the caller's team. Secret-bearing output"
|
||||
+ " options are returned as a redaction sentinel, never their stored"
|
||||
+ " values.")
|
||||
public List<Policy> listPolicies() {
|
||||
return policyAccessGuard.visibleFrom(policyStore);
|
||||
return policyAccessGuard.visibleFrom(policyStore).stream()
|
||||
.map(PolicyController::withMaskedOutputSecrets)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@GetMapping("/overview")
|
||||
@@ -337,11 +403,17 @@ public class PolicyController {
|
||||
}
|
||||
|
||||
@GetMapping("/{policyId}")
|
||||
@Operation(summary = "Get a policy by id")
|
||||
@Operation(
|
||||
summary = "Get a policy by id",
|
||||
description =
|
||||
"Secret-bearing output options are returned as a redaction sentinel, never"
|
||||
+ " their stored values; an edit that sends the sentinel back keeps"
|
||||
+ " them.")
|
||||
public ResponseEntity<Policy> getPolicy(@PathVariable String policyId) {
|
||||
return policyStore
|
||||
.get(policyId)
|
||||
.filter(policyAccessGuard::canAccess)
|
||||
.map(PolicyController::withMaskedOutputSecrets)
|
||||
.map(ResponseEntity::ok)
|
||||
.orElseGet(() -> ResponseEntity.notFound().build());
|
||||
}
|
||||
@@ -401,6 +473,7 @@ public class PolicyController {
|
||||
() ->
|
||||
new ResponseStatusException(
|
||||
HttpStatus.NOT_FOUND, "No policy: " + policyId));
|
||||
stampPolicyAudit(policy.toDefinition());
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId();
|
||||
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
|
||||
@@ -412,9 +485,10 @@ public class PolicyController {
|
||||
description =
|
||||
"Pulls the policy's configured sources and runs the pipeline now, regardless of"
|
||||
+ " the enabled flag (which only gates automatic triggering). Returns"
|
||||
+ " the ids of the runs started; poll the run-status endpoint for each."
|
||||
+ " Empty when the sources yielded no work to do.")
|
||||
public ResponseEntity<List<String>> trigger(@PathVariable String policyId) {
|
||||
+ " the ids of the runs started (poll the run-status endpoint for each)"
|
||||
+ " plus what the sweep skipped - already-processed, parked-by-failure,"
|
||||
+ " and in-flight counts - so an empty result explains itself.")
|
||||
public ResponseEntity<SweepOutcome> trigger(@PathVariable String policyId) {
|
||||
Policy policy =
|
||||
policyStore
|
||||
.get(policyId)
|
||||
@@ -433,6 +507,60 @@ public class PolicyController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stamp this run's policy name and step endpoints onto the current request so the audit aspect
|
||||
* can label the event as the policy it ran (rather than the generic {@code /run} endpoint) and
|
||||
* record which tools it executed. No-op outside a web request.
|
||||
*/
|
||||
private static void stampPolicyAudit(PipelineDefinition definition) {
|
||||
if (definition == null
|
||||
|| !(RequestContextHolder.getRequestAttributes()
|
||||
instanceof ServletRequestAttributes attrs)) {
|
||||
return;
|
||||
}
|
||||
HttpServletRequest request = attrs.getRequest();
|
||||
if (definition.name() != null && !definition.name().isBlank()) {
|
||||
request.setAttribute(AuditContext.REQ_ATTR_POLICY_NAME, definition.name());
|
||||
}
|
||||
List<String> steps =
|
||||
definition.steps().stream()
|
||||
.map(PipelineStep::operation)
|
||||
.filter(op -> op != null && !op.isBlank())
|
||||
.toList();
|
||||
if (!steps.isEmpty()) {
|
||||
request.setAttribute(AuditContext.REQ_ATTR_POLICY_STEPS, steps);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Authorization-check an ad-hoc run's output while the caller's principal is present (this
|
||||
* request thread). The worker thread that later delivers carries no security context, so an S3
|
||||
* output's connection-access check would be skipped there; without this gate a caller could
|
||||
* reference another tenant's connection by id and write to it (confused deputy). Stored
|
||||
* policies are covered by save-time {@link PolicyValidator#validate} instead.
|
||||
*/
|
||||
private void validateAdHocOutput(PipelineDefinition definition) {
|
||||
if (definition.output() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
policyValidator.validateOutput(definition.output());
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ad-hoc runs (AI / one-off pipelines) are still editor activity, so their supplied documents
|
||||
* feed the same virtual editor source as stored editor policies, counted against the caller's
|
||||
* team. A run with no primary documents (generator pipeline) records nothing.
|
||||
*/
|
||||
private void recordEditorDocs(PolicyInputs inputs) {
|
||||
docCounter.record(
|
||||
EditorSource.counterKey(sourceAccessGuard.currentTeamId()),
|
||||
inputs.primary().size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn the typed run files into engine {@link PolicyInputs}: the primary documents plus the
|
||||
* named supporting-file store, where each asset's {@code key} is the name a step references
|
||||
|
||||
@@ -21,7 +21,9 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.common.service.AutomationRunContext;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.service.InternalApiTimeoutException;
|
||||
import stirling.software.common.service.JobOwnershipService;
|
||||
import stirling.software.common.service.JobQueue;
|
||||
@@ -148,6 +150,7 @@ public class PolicyEngine {
|
||||
runAsPrincipal(
|
||||
billingPrincipal,
|
||||
fileOwner,
|
||||
definition.name(),
|
||||
() -> runToCompletion(run, inputs, tracking, completion));
|
||||
|
||||
// One admission unit per run; steps run synchronously within it, so this gates heavy work
|
||||
@@ -199,63 +202,78 @@ public class PolicyEngine {
|
||||
PolicyProgressListener listener,
|
||||
CompletableFuture<PolicyRun> completion) {
|
||||
String runId = run.getRunId();
|
||||
try {
|
||||
run.markRunning();
|
||||
PolicyExecutionResult result =
|
||||
stepExecutor.execute(run.getDefinition(), inputs, listener);
|
||||
OutputSpec output = run.getDefinition().output();
|
||||
List<ResultFile> outputs =
|
||||
sinkFor(output)
|
||||
.deliver(
|
||||
new OutputDelivery(runId, run.getPolicyId()),
|
||||
result.files(),
|
||||
output);
|
||||
taskManager.setMultipleFileResults(runId, outputs);
|
||||
taskManager.setComplete(runId);
|
||||
run.complete(outputs);
|
||||
} catch (PolicyInputRequiredException e) {
|
||||
// Expected path: suspend rather than fail. Persist intermediates as fileIds so the run
|
||||
// can resume after this worker thread is gone.
|
||||
WaitState wait = suspend(e);
|
||||
run.waitForInput(wait);
|
||||
taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
String message = toolTimeoutMessage(e);
|
||||
log.error(
|
||||
"Policy run {} timed out on {}: {}",
|
||||
runId,
|
||||
e.getEndpointPath(),
|
||||
e.getMessage());
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
} catch (RestClientResponseException e) {
|
||||
// A downstream tool call returned an error status. When it's a structured entitlement
|
||||
// response (401/402 with a JSON `error` sentinel), surface that code onto the run so
|
||||
// the
|
||||
// client can react — e.g. pop the usage-limit modal — instead of only seeing a generic
|
||||
// failure. We don't interpret the code here (that would couple this module to the saas
|
||||
// billing layer); we just pass it through for the client to map. Other statuses fall
|
||||
// through to the generic failure below.
|
||||
String code = DownstreamEntitlementError.extractCode(e);
|
||||
if (code != null) {
|
||||
log.info("Policy run {} blocked by downstream entitlement gate ({})", runId, code);
|
||||
String message = "Usage limit reached";
|
||||
run.failWithCode(message, code, DownstreamEntitlementError.extractSubscribed(e));
|
||||
taskManager.setError(runId, message);
|
||||
} else {
|
||||
String message = "Policy run failed: " + e.getMessage();
|
||||
log.error("Policy run {} failed (downstream HTTP error)", runId, e);
|
||||
// One policy run = one automation run. Scope the run id on this worker thread (the async
|
||||
// hop already happened) so every tool sub-step dispatched via InternalApiClient groups into
|
||||
// a single charge, and two separate policy runs on the same document stay distinct charges.
|
||||
try (AutomationRunContext.Scope runScope = AutomationRunContext.open(runId)) {
|
||||
try {
|
||||
run.markRunning();
|
||||
PolicyExecutionResult result =
|
||||
stepExecutor.execute(run.getDefinition(), inputs, listener);
|
||||
OutputSpec output = run.getDefinition().output();
|
||||
List<ResultFile> outputs =
|
||||
sinkFor(output)
|
||||
.deliver(
|
||||
new OutputDelivery(runId, run.getPolicyId()),
|
||||
result.files(),
|
||||
output);
|
||||
taskManager.setMultipleFileResults(runId, outputs);
|
||||
taskManager.setComplete(runId);
|
||||
run.complete(outputs);
|
||||
} catch (PolicyInputRequiredException e) {
|
||||
// Expected path: suspend rather than fail. Persist intermediates as fileIds so the
|
||||
// run
|
||||
// can resume after this worker thread is gone.
|
||||
WaitState wait = suspend(e);
|
||||
run.waitForInput(wait);
|
||||
taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
String message = toolTimeoutMessage(e);
|
||||
log.error(
|
||||
"Policy run {} timed out on {}: {}",
|
||||
runId,
|
||||
e.getEndpointPath(),
|
||||
e.getMessage());
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
} catch (RestClientResponseException e) {
|
||||
// A downstream tool call returned an error status. When it's a structured
|
||||
// entitlement
|
||||
// response (401/402 with a JSON `error` sentinel), surface that code onto the run
|
||||
// so
|
||||
// the
|
||||
// client can react — e.g. pop the usage-limit modal — instead of only seeing a
|
||||
// generic
|
||||
// failure. We don't interpret the code here (that would couple this module to the
|
||||
// saas
|
||||
// billing layer); we just pass it through for the client to map. Other statuses
|
||||
// fall
|
||||
// through to the generic failure below.
|
||||
String code = DownstreamEntitlementError.extractCode(e);
|
||||
if (code != null) {
|
||||
log.info(
|
||||
"Policy run {} blocked by downstream entitlement gate ({})",
|
||||
runId,
|
||||
code);
|
||||
String message = "Usage limit reached";
|
||||
run.failWithCode(
|
||||
message, code, DownstreamEntitlementError.extractSubscribed(e));
|
||||
taskManager.setError(runId, message);
|
||||
} else {
|
||||
String message = "Policy run failed: " + e.getMessage();
|
||||
log.error("Policy run {} failed (downstream HTTP error)", runId, e);
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
String message = "Policy run failed: " + e.getMessage();
|
||||
log.error("Policy run {} failed", runId, e);
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
} finally {
|
||||
// Always resolve so stream/await callers unblock.
|
||||
completion.complete(run);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
String message = "Policy run failed: " + e.getMessage();
|
||||
log.error("Policy run {} failed", runId, e);
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
} finally {
|
||||
// Always resolve so stream/await callers unblock.
|
||||
completion.complete(run);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,18 +379,26 @@ public class PolicyEngine {
|
||||
* dispatch attributes (and charges) usage to that user. A null/blank principal runs as-is.
|
||||
* Restores the previous MDC value afterward (defensive — worker threads aren't pooled).
|
||||
*/
|
||||
private static void runAsPrincipal(String billingPrincipal, String fileOwner, Runnable body) {
|
||||
private static void runAsPrincipal(
|
||||
String billingPrincipal, String fileOwner, String policyName, Runnable body) {
|
||||
// Billing identity (MDC auditPrincipal) and output-file ownership (JobContext owner) are
|
||||
// set
|
||||
// independently: usage is charged to billingPrincipal, but stored output files are owned by
|
||||
// fileOwner — the user who triggered an org-wide policy — so they can fetch their results.
|
||||
// Either may be null (e.g. login disabled, or a trigger-fired run); each is applied only
|
||||
// when present and restored afterward (defensive — worker threads aren't pooled).
|
||||
// when present and restored afterward (defensive — worker threads aren't pooled). The
|
||||
// policy
|
||||
// name rides MDC too so each tool step's loopback dispatch (InternalApiClient) can forward
|
||||
// it as a header, letting the audit tie the step back to its policy.
|
||||
String previousPrincipal = MDC.get(AUDIT_PRINCIPAL_MDC_KEY);
|
||||
String previousPolicyName = MDC.get(InternalApiClient.POLICY_NAME_MDC_KEY);
|
||||
String previousOwner = JobContext.getOwner();
|
||||
if (billingPrincipal != null && !billingPrincipal.isBlank()) {
|
||||
MDC.put(AUDIT_PRINCIPAL_MDC_KEY, billingPrincipal);
|
||||
}
|
||||
if (policyName != null && !policyName.isBlank()) {
|
||||
MDC.put(InternalApiClient.POLICY_NAME_MDC_KEY, policyName);
|
||||
}
|
||||
if (fileOwner != null && !fileOwner.isBlank()) {
|
||||
JobContext.setOwner(fileOwner);
|
||||
}
|
||||
@@ -384,6 +410,11 @@ public class PolicyEngine {
|
||||
} else {
|
||||
MDC.remove(AUDIT_PRINCIPAL_MDC_KEY);
|
||||
}
|
||||
if (previousPolicyName != null) {
|
||||
MDC.put(InternalApiClient.POLICY_NAME_MDC_KEY, previousPolicyName);
|
||||
} else {
|
||||
MDC.remove(InternalApiClient.POLICY_NAME_MDC_KEY);
|
||||
}
|
||||
JobContext.setOwner(previousOwner);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ public class PolicyExecutor {
|
||||
// One call over all inputs. The outputs derive from a single input only when exactly
|
||||
// one entered; otherwise (a genuine merge) there is no single source.
|
||||
ToolResult r = callEndpoint(step, inputFiles, supportingFiles);
|
||||
Integer origin = inputOrigins.size() == 1 ? inputOrigins.getFirst() : null;
|
||||
Integer origin = inputOrigins.size() == 1 ? inputOrigins.get(0) : null;
|
||||
for (Resource file : r.files()) {
|
||||
files.add(file);
|
||||
origins.add(origin);
|
||||
|
||||
@@ -21,6 +21,7 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
import stirling.software.proprietary.policy.model.PolicyRunStatus;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
import stirling.software.proprietary.policy.source.EditorSource;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceDocCounter;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
@@ -44,7 +45,7 @@ public class PolicyRunner {
|
||||
private final ProcessedLedger processedLedger;
|
||||
|
||||
/** Full-listing sweep: resolve every source, then reconcile the ledger. */
|
||||
public List<String> run(Policy policy) {
|
||||
public SweepOutcome run(Policy policy) {
|
||||
return run(policy, SweepKind.FULL);
|
||||
}
|
||||
|
||||
@@ -52,10 +53,10 @@ public class PolicyRunner {
|
||||
* Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
|
||||
* one failure does not affect the others. No sources means one run with no input (generator
|
||||
* pipeline). Missing or disabled sources are skipped so one broken reference does not stop the
|
||||
* rest. Returns the ids of the runs it started (empty when sources yielded no work), so a
|
||||
* manual trigger can report back which runs to follow.
|
||||
* rest. Returns the ids of the runs it started plus what the sweep skipped, so a manual trigger
|
||||
* can report which runs to follow or why nothing ran.
|
||||
*/
|
||||
public List<String> run(Policy policy, SweepKind sweep) {
|
||||
public SweepOutcome run(Policy policy, SweepKind sweep) {
|
||||
long sweepStart = System.currentTimeMillis();
|
||||
PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger);
|
||||
List<String> runIds = new ArrayList<>();
|
||||
@@ -95,13 +96,19 @@ public class PolicyRunner {
|
||||
policy.id());
|
||||
}
|
||||
}
|
||||
return runIds;
|
||||
return context.outcome(runIds);
|
||||
}
|
||||
|
||||
/** Run a stored policy on caller-supplied files (e.g. manual upload), bypassing its sources. */
|
||||
/**
|
||||
* Run a stored policy on caller-supplied files (e.g. an editor upload), bypassing its sources.
|
||||
* The supplied documents are still counted against the virtual {@link EditorSource}, scoped to
|
||||
* the policy's team, so the Sources overview reports the whole team's editor throughput.
|
||||
*/
|
||||
public PolicyRunHandle runWith(
|
||||
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
|
||||
return policyEngine.runPolicy(policy, inputs, listener);
|
||||
PolicyRunHandle handle = policyEngine.runPolicy(policy, inputs, listener);
|
||||
docCounter.record(EditorSource.counterKey(policy.teamId()), inputs.primary().size());
|
||||
return handle;
|
||||
}
|
||||
|
||||
/** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */
|
||||
|
||||
@@ -86,4 +86,32 @@ final class PolicySweep implements ResolveContext {
|
||||
synchronized Set<String> presentIdentities() {
|
||||
return Set.copyOf(present);
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarise the sweep from state already in hand (no extra ledger reads): the prefetched rows
|
||||
* were loaded before claiming, and successful claims flipped their entries to PROCESSING, so
|
||||
* what remains DONE or ERROR is exactly what this sweep skipped.
|
||||
*/
|
||||
synchronized SweepOutcome outcome(List<String> runIds) {
|
||||
int alreadyProcessed = 0;
|
||||
int parked = 0;
|
||||
int processing = 0;
|
||||
for (String identity : present) {
|
||||
ClaimState state = prefetched.get(identity);
|
||||
if (state == null) {
|
||||
continue;
|
||||
}
|
||||
switch (state.status()) {
|
||||
case DONE -> alreadyProcessed++;
|
||||
case ERROR -> parked++;
|
||||
case PROCESSING, INTERRUPTED -> processing++;
|
||||
}
|
||||
}
|
||||
return new SweepOutcome(
|
||||
runIds,
|
||||
present.size(),
|
||||
alreadyProcessed,
|
||||
parked,
|
||||
Math.max(0, processing - runIds.size()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,19 @@ public class PolicyValidator {
|
||||
InputSpec spec = source.toInputSpec();
|
||||
inputSourceFor(spec).validate(spec);
|
||||
}
|
||||
outputSinkFor(policy.output()).validate(policy.output());
|
||||
validateOutput(policy.output());
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an output spec against its sink. Must be called on a request thread (caller's
|
||||
* principal present) so an S3 output's connection is authorization-checked against the caller -
|
||||
* ad-hoc runs are never persisted and so never hit {@link #validate(Policy)}, and the worker
|
||||
* thread that later delivers has no principal, so this is their only access gate.
|
||||
*
|
||||
* @throws IllegalArgumentException if the type is unknown or the config is invalid/inaccessible
|
||||
*/
|
||||
public void validateOutput(OutputSpec output) {
|
||||
outputSinkFor(output).validate(output);
|
||||
}
|
||||
|
||||
private PolicyTrigger triggerFor(TriggerConfig config) {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* What one policy sweep found and started, so a manual trigger can explain an empty result instead
|
||||
* of a blanket "nothing to do": how many files the sources listed, how many were skipped because
|
||||
* they are already processed at their current version, how many are parked by a failed run (not
|
||||
* retried until they change or history is cleared), and how many are still in flight from an
|
||||
* earlier sweep. Counts are zero for {@link SweepKind#LIGHT} sweeps, which do not take a full
|
||||
* listing.
|
||||
*/
|
||||
public record SweepOutcome(
|
||||
List<String> runIds, int filesListed, int alreadyProcessed, int parked, int inFlight) {
|
||||
|
||||
public SweepOutcome {
|
||||
runIds = runIds == null ? List.of() : List.copyOf(runIds);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.core.io.AbstractResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.s3.S3Config;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.s3.S3Identities;
|
||||
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
||||
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
|
||||
/**
|
||||
* Reads input files from an Amazon S3 (or S3-compatible) bucket; each listed object is its own unit
|
||||
* of work, claimed through the {@link ResolveContext} ledger and tracked in place. Identity and
|
||||
* version gate come from {@link S3Identities}, so the steady-state sweep never downloads content.
|
||||
* Options: "connectionId" references the stored S3 connection (an {@code IntegrationConfig} owning
|
||||
* bucket, region, endpoint, and credentials - resolved by {@link S3ConnectionResolver}); "prefix"
|
||||
* (only keys starting with it are read) and "mode" are per-source, where mode is "consume"
|
||||
* (default: a processed object is deleted once every policy that claimed it has settled
|
||||
* successfully and it is still the version that ran; failures stay in place and are not retried
|
||||
* until they change) or "snapshot" (stateless, every run sees the full set). Keys ending in "/"
|
||||
* (folder placeholders) and keys with a dot-prefixed path segment are never picked up, mirroring
|
||||
* the folder source's hidden-file rule.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class S3InputSource implements InputSource {
|
||||
|
||||
private static final String TYPE = "s3";
|
||||
|
||||
private final S3ConnectionPool connectionPool;
|
||||
private final S3ConnectionResolver connectionResolver;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(InputSpec spec) {
|
||||
return spec != null && TYPE.equals(spec.type());
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails fast at save time: an unknown/disabled/unusable connection, bad config shape, a private
|
||||
* endpoint without the operator opt-in, or a bucket the connection cannot list.
|
||||
*/
|
||||
@Override
|
||||
public void validate(InputSpec spec) {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
try {
|
||||
connectionPool.clientFor(config).listObjectsV2(listRequest(config).maxKeys(1).build());
|
||||
} catch (SdkException e) {
|
||||
throw new IllegalArgumentException(
|
||||
"cannot access s3://"
|
||||
+ config.bucket()
|
||||
+ "/"
|
||||
+ config.prefix()
|
||||
+ ": "
|
||||
+ e.getMessage(),
|
||||
e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Client client = connectionPool.clientFor(config);
|
||||
// A listing failure propagates so the sweep reads it as "could not list" (which vetoes
|
||||
// presence cleanup), never as "verifiably no objects".
|
||||
List<S3Object> objects = listObjects(client, config);
|
||||
|
||||
if (config.snapshot()) {
|
||||
return objects.stream()
|
||||
.map(
|
||||
object ->
|
||||
ResolvedInput.of(
|
||||
PolicyInputs.of(
|
||||
List.of(
|
||||
objectResource(
|
||||
client, config, object)))))
|
||||
.toList();
|
||||
}
|
||||
|
||||
ctx.reportPresent(
|
||||
objects.stream()
|
||||
.map(object -> S3Identities.identity(config.bucket(), object.key()))
|
||||
.toList());
|
||||
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (S3Object object : objects) {
|
||||
String identity = S3Identities.identity(config.bucket(), object.key());
|
||||
String gate = S3Identities.gate(object.eTag(), object.size(), object.lastModified());
|
||||
if (!ctx.claim(identity, gate, null)) {
|
||||
continue;
|
||||
}
|
||||
work.add(
|
||||
new ResolvedInput(
|
||||
PolicyInputs.of(List.of(objectResource(client, config, object))),
|
||||
success ->
|
||||
completeConsumed(
|
||||
ctx,
|
||||
client,
|
||||
config,
|
||||
object.key(),
|
||||
identity,
|
||||
gate,
|
||||
success)));
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
/**
|
||||
* Settle at the version this run claimed, then remove the object only when it still carries
|
||||
* that version and every policy that claimed it has settled DONE, mirroring the folder source's
|
||||
* consensus delete. A failed run settles ERROR and never deletes; the DONE row of an object
|
||||
* that could not be deleted still stops reprocessing.
|
||||
*/
|
||||
private void completeConsumed(
|
||||
ResolveContext ctx,
|
||||
S3Client client,
|
||||
S3Config config,
|
||||
String key,
|
||||
String identity,
|
||||
String claimGate,
|
||||
boolean success) {
|
||||
ctx.settle(identity, claimGate, null, success);
|
||||
if (!success) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
HeadObjectResponse head =
|
||||
client.headObject(
|
||||
HeadObjectRequest.builder().bucket(config.bucket()).key(key).build());
|
||||
String currentGate =
|
||||
S3Identities.gate(head.eTag(), head.contentLength(), head.lastModified());
|
||||
if (currentGate.equals(claimGate) && ctx.allSettledDone(identity)) {
|
||||
client.deleteObject(
|
||||
DeleteObjectRequest.builder().bucket(config.bucket()).key(key).build());
|
||||
}
|
||||
} catch (NoSuchKeyException alreadyGone) {
|
||||
// Removed by the user or a co-watching policy's own consensus delete: nothing to do.
|
||||
} catch (S3Exception e) {
|
||||
if (e.statusCode() == 404) {
|
||||
return;
|
||||
}
|
||||
log.warn("Could not remove consumed S3 object {}: {}", identity, e.getMessage());
|
||||
} catch (SdkException e) {
|
||||
log.warn("Could not remove consumed S3 object {}: {}", identity, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** Every ingestible object under the configured prefix, across all listing pages. */
|
||||
private static List<S3Object> listObjects(S3Client client, S3Config config) {
|
||||
List<S3Object> objects = new ArrayList<>();
|
||||
String continuationToken = null;
|
||||
do {
|
||||
ListObjectsV2Request.Builder request = listRequest(config);
|
||||
if (continuationToken != null) {
|
||||
request.continuationToken(continuationToken);
|
||||
}
|
||||
ListObjectsV2Response page = client.listObjectsV2(request.build());
|
||||
for (S3Object object : page.contents()) {
|
||||
if (ingestible(object)) {
|
||||
objects.add(object);
|
||||
}
|
||||
}
|
||||
continuationToken = page.nextContinuationToken();
|
||||
} while (continuationToken != null);
|
||||
return objects;
|
||||
}
|
||||
|
||||
private static ListObjectsV2Request.Builder listRequest(S3Config config) {
|
||||
ListObjectsV2Request.Builder request =
|
||||
ListObjectsV2Request.builder().bucket(config.bucket());
|
||||
if (!config.prefix().isEmpty()) {
|
||||
request.prefix(config.prefix());
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* Folder-placeholder keys (ending "/") and keys with a dot-prefixed segment are skipped, so a
|
||||
* hidden convention (e.g. a future output sink's staging prefix) is never re-ingested.
|
||||
*/
|
||||
private static boolean ingestible(S3Object object) {
|
||||
String key = object.key();
|
||||
if (key.isEmpty() || key.endsWith("/")) {
|
||||
return false;
|
||||
}
|
||||
for (String segment : key.split("/")) {
|
||||
if (segment.startsWith(".")) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Resource objectResource(S3Client client, S3Config config, S3Object object) {
|
||||
return new S3ObjectResource(client, config.bucket(), object);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams the object on demand, pinned to the ETag observed at listing time so a run never
|
||||
* reads a different version than the sweep claimed (a swapped object fails the read with a
|
||||
* precondition error and the new version is claimed by a later sweep).
|
||||
*/
|
||||
private static final class S3ObjectResource extends AbstractResource {
|
||||
|
||||
private final S3Client client;
|
||||
private final String bucket;
|
||||
private final String key;
|
||||
private final String eTag;
|
||||
private final Long size;
|
||||
|
||||
private S3ObjectResource(S3Client client, String bucket, S3Object object) {
|
||||
this.client = client;
|
||||
this.bucket = bucket;
|
||||
this.key = object.key();
|
||||
this.eTag = object.eTag();
|
||||
this.size = object.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getInputStream() throws IOException {
|
||||
GetObjectRequest.Builder request = GetObjectRequest.builder().bucket(bucket).key(key);
|
||||
if (eTag != null && !eTag.isBlank()) {
|
||||
request.ifMatch(eTag);
|
||||
}
|
||||
try {
|
||||
return client.getObject(request.build());
|
||||
} catch (NoSuchKeyException e) {
|
||||
throw new FileNotFoundException(getDescription() + " no longer exists");
|
||||
} catch (SdkException e) {
|
||||
throw new IOException(
|
||||
"Could not read " + getDescription() + ": " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Listed just now; readers get a precise error from {@link #getInputStream} instead. */
|
||||
@Override
|
||||
public boolean exists() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long contentLength() {
|
||||
return size == null ? -1 : size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return key.substring(key.lastIndexOf('/') + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "S3 object " + S3Identities.identity(bucket, key);
|
||||
}
|
||||
}
|
||||
}
|
||||