Compare commits

..
Author SHA1 Message Date
Ludy87 7265d1b9fa Update PolicyExecutor.java 2026-07-09 22:52:21 +02:00
Ludy87 a1808b6c0f Update PortalInfraAuditService.java 2026-07-09 22:51:55 +02:00
LudyandGitHub e78b8fcd37 Merge branch 'main' into switch_expression 2026-07-09 22:41:35 +02:00
LudyandGitHub e263b9f507 Merge branch 'main' into switch_expression 2026-07-09 11:06:15 +02:00
Ludy87 16ac5e4822 Update MergeControllerGapTest.java 2026-07-03 16:15:00 +02:00
Ludy87 b4a1104902 more 2026-07-03 16:08:50 +02:00
LudyandGitHub 2581c11c9d Merge branch 'main' into switch_expression 2026-07-03 15:51:17 +02:00
LudyandGitHub b64f626926 Merge branch 'main' into switch_expression 2026-07-03 15:07:11 +02:00
Ludy87 76b129decb Remove unified credit interceptor system
Delete CreditErrorAdvice, UnifiedCreditInterceptor, and TeamCreditService. These classes implemented credit validation and consumption logic for the SaaS profile. This removal indicates a refactoring or replacement of the credit management system.
2026-07-03 15:06:49 +02:00
LudyandGitHub 6810155799 Merge branch 'main' into switch_expression 2026-06-14 17:33:00 +02:00
LudyandGitHub fe520d24a9 Delete CLAUDE.md~Stashed changes 2026-06-14 17:31:12 +02:00
Ludy87 351abf54a1 Warn on null form field instead of setting value
Split the combined 'case null, default' switch arm into distinct cases. Now a null PDF form field triggers a warning log, while the default branch continues to call setValue(value != null ? value : ""). This improves diagnostics and avoids attempting to set values on null fields.
2026-06-13 22:13:16 +02:00
Ludy87 cad2ac206a Update UIDataController.java 2026-06-12 13:27:58 +02:00
Ludy87 4598ae1e8d Use constructor injection in UploadLimitServiceTest
Replace reflective field injection with passing the mocked ApplicationProperties into UploadLimitService via its constructor. This simplifies the test setup, avoids reflection, and makes dependency injection explicit and more readable.
2026-06-12 13:14:41 +02:00
Ludy87 a6a6de960e Refactor switch statements to use switch expressions for improved readability and conciseness
Signed-off-by: Ludy87 <Ludy87@users.noreply.github.com>
2026-06-12 12:58:14 +02:00
LudyandGitHub 9a4af79c6d Merge branch 'main' into switch_expression 2026-06-12 12:43:49 +02:00
Ludy87 b43cd350f7 Merge branch 'Stirling-Tools:main' into main
Signed-off-by: Ludy87 <Ludy87@users.noreply.github.com>
2026-06-12 12:43:33 +02:00
LudyandGitHub 9a985abf46 Merge branch 'main' into switch_expression 2026-06-11 20:38:42 +02:00
Ludy87 837371cff0 del 2026-06-11 20:38:16 +02:00
Ludy87 da7b586313 Use deque accessors and add job ownership guard
Replace index-based List access with deque-style getFirst()/getLast() where appropriate, modernize switches to expression form, and update various small APIs (e.g. getFirst() on list-like structures). Remove an obsolete test file. Substantially enhance JobController to integrate cluster backplane and job store: add ownership cache, guardNonOwner/backplaneUnavailable helpers, sticky-410 handling, and backplane-aware flows for job status/result/file endpoints (returning 410 or retryable 503 as needed). Update tests and PDF-to-markdown flow to use PdfMarkdownConverter/TempFile/PdfDocument mocks. Misc fixes across controllers and services (SAML attribute access, audit file access, database backup handling, session registry, interceptors, and team credit service). These changes improve safety for clustered deployments and modernize collection usage and control flow.
2026-06-11 20:36:12 +02:00
Ludy87 33bfb18d7c Merge remote-tracking branch 'origin/main' into switch_expression 2026-06-11 20:28:12 +02:00
Ludy87 584fd0cb09 getFirst() 2026-05-09 20:44:13 +02:00
Ludy87 ecdb8160a9 getLast 2026-05-09 20:32:21 +02:00
Ludy87 0bd7d72d07 get(0) to getFirst 2026-05-09 20:20:12 +02:00
Ludy87 10eac6622c UIDataControllerTest 2026-05-09 20:14:25 +02:00
Ludy87 641114c5c2 expression 2026-05-09 20:11:24 +02:00
650 changed files with 10841 additions and 109331 deletions
+5 -5
View File
@@ -1,12 +1,12 @@
build: &build
- build.gradle
- app/(common|core|proprietary|saas)/build.gradle
- app/(common|core|proprietary)/build.gradle
- Taskfile.yml
- .taskfiles/backend.yml
openapi: &openapi
- *build
- app/(common|core|proprietary|saas)/src/main/java/**
- app/(common|core|proprietary)/src/main/java/**
docker-base: &docker-base
- docker/base/Dockerfile
@@ -23,9 +23,9 @@ docker: &docker
- *docker-base
project: &project
- app/(common|core|proprietary|saas)/src/(main|test)/java/**
- app/(common|core|proprietary)/src/(main|test)/java/**
- *build
- "app/(common|core|proprietary|saas)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*"
- "app/(common|core|proprietary)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*"
- exampleYmlFiles/**
- gradle/**
- libs/**
@@ -82,7 +82,7 @@ tauri: &tauri
# tool surfaces it generates models from.
engine: &engine
- engine/**
- app/(common|core|proprietary|saas)/src/main/java/**
- app/(common|core|proprietary)/src/main/java/**
- .github/workflows/ai-engine.yml
- Taskfile.yml
- .taskfiles/engine.yml
-7
View File
@@ -63,7 +63,6 @@ labels:
files:
- 'app/core/src/main/resources/static/.*'
- 'app/proprietary/src/main/resources/static/.*'
- 'app/saas/src/main/resources/static/.*'
- 'frontend/**'
- 'frontend/.*'
- 'frontend/**/.*'
@@ -84,7 +83,6 @@ labels:
- 'app/common/src/main/java/.*.java'
- 'app/proprietary/src/main/java/.*.java'
- 'app/core/src/main/java/.*.java'
- 'app/saas/src/main/java/.*.java'
- label: 'Back End'
files:
@@ -92,9 +90,6 @@ labels:
- 'app/core/src/main/java/stirling/software/SPDF/controller/.*'
- 'app/core/src/main/resources/settings.yml.template'
- 'app/core/src/main/resources/application.properties'
- 'app/proprietary/src/main/resources/application-proprietary.properties'
- 'app/saas/src/main/resources/application-dev.properties'
- 'app/saas/src/main/resources/application-saas.properties'
- 'app/core/src/main/resources/banner.txt'
- 'app/core/src/main/resources/static/python/png_to_webp.py'
- 'app/core/src/main/resources/static/python/split_photos.py'
@@ -158,7 +153,6 @@ labels:
- 'app/common/src/test/.*'
- 'app/proprietary/src/test/.*'
- 'app/core/src/test/.*'
- 'app/saas/src/test/.*'
- 'testing/.*'
- '.github/workflows/scorecards.yml'
- 'exampleYmlFiles/test_cicd.yml'
@@ -177,4 +171,3 @@ labels:
- 'app/common/build.gradle'
- 'app/proprietary/build.gradle'
- 'app/core/build.gradle'
- 'app/saas/build.gradle'
+1 -1
View File
@@ -256,7 +256,7 @@ jobs:
- name: Build and push V2 image (Docker fork fallback)
if: env.USE_DEPOT != 'true' && steps.check-image.outputs.exists == 'false'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
file: ./docker/embedded/Dockerfile
@@ -270,7 +270,7 @@ jobs:
- name: Build and push PR-specific image (Docker fork fallback)
if: env.USE_DEPOT != 'true'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
file: ./docker/embedded/Dockerfile
@@ -296,7 +296,7 @@ jobs:
- name: Build and push engine image (Docker fork fallback)
if: env.USE_DEPOT != 'true' && needs.check-comment.outputs.enable_prototypes == 'true'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: ./engine
file: ./engine/Dockerfile
+1 -1
View File
@@ -47,7 +47,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
+1 -1
View File
@@ -29,7 +29,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
+1 -1
View File
@@ -34,7 +34,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
+1 -1
View File
@@ -51,7 +51,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
distribution: temurin
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
+2 -2
View File
@@ -121,7 +121,7 @@ jobs:
- name: Build and push frontend image (Docker fork fallback)
if: env.USE_DEPOT != 'true' && steps.check-frontend.outputs.exists == 'false'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
file: ./docker/frontend/Dockerfile
@@ -150,7 +150,7 @@ jobs:
- name: Build and push backend image (Docker fork fallback)
if: env.USE_DEPOT != 'true' && steps.check-backend.outputs.exists == 'false'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
file: ./docker/backend/Dockerfile
+1 -1
View File
@@ -50,7 +50,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
+1 -1
View File
@@ -61,7 +61,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependencies
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/caches
+1 -1
View File
@@ -85,7 +85,7 @@ jobs:
- name: Build and push base image
id: build-push-base
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: docker/base
+5 -5
View File
@@ -66,7 +66,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependencies
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/caches
@@ -145,7 +145,7 @@ jobs:
id: build-push-latest
# Empty-tag guard: build-push-action errors when asked to push with no tags.
if: env.RUN_MAIN_APP == 'true' && steps.meta.outputs.tags != ''
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
@@ -192,7 +192,7 @@ jobs:
- name: Build and push Unified Dockerfile (fat variant)
id: build-push-fat
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-fat.outputs.tags != ''
with:
builder: ${{ steps.buildx.outputs.name }}
@@ -236,7 +236,7 @@ jobs:
- name: Build and push Unified Dockerfile (ultra-lite variant)
id: build-push-lite
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-lite.outputs.tags != ''
with:
builder: ${{ steps.buildx.outputs.name }}
@@ -365,7 +365,7 @@ jobs:
- name: Build and push unoserver image
id: build-push-unoserver
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode != 'skip'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
+3 -3
View File
@@ -95,7 +95,7 @@ jobs:
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
@@ -186,7 +186,7 @@ jobs:
# (multi-platform + gha cache) against the published base image.
- name: Build ${{ matrix.docker-rev }} (Docker fork fallback)
if: env.USE_DEPOT != 'true' && inputs.docker-base-changed != 'true'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
@@ -259,7 +259,7 @@ jobs:
- name: Build docker/unoserver/Dockerfile (Docker fork fallback)
if: env.USE_DEPOT != 'true'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
+1 -1
View File
@@ -95,7 +95,7 @@ jobs:
- name: Build and push test image (Docker fork fallback)
if: env.USE_DEPOT != 'true'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
file: ./docker/embedded/Dockerfile
-5
View File
@@ -1,5 +0,0 @@
{
"ignoredFiles": [
"frontend/editor/src-tauri/icons/icon.png"
]
}
-6
View File
@@ -80,12 +80,6 @@ tasks:
OPEN: '{{.OPEN | default ""}}'
env:
BACKEND_URL: '{{.BACKEND_URL}}'
# Dev-only browser-tab label so concurrent worktrees are distinguishable.
# Only the worktree folder basename (e.g. "wt1") is exposed — never the
# full path, hostname, or user. Consumed at dev-serve time by vite.config
# and dropped from production builds.
STIRLING_DEV_LABEL:
sh: basename "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
cmds:
- npx vite editor --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}}
@@ -73,7 +73,7 @@ public class RuntimePathConfig {
defaultWatchedFolders,
watchedFoldersDirs,
pipeline != null ? pipeline.getWatchedFoldersDir() : null);
this.pipelineWatchedFoldersPath = this.pipelineWatchedFoldersPaths.get(0);
this.pipelineWatchedFoldersPath = this.pipelineWatchedFoldersPaths.getFirst();
this.pipelineFinishedFoldersPath =
resolvePath(
defaultFinishedFolders,
@@ -246,17 +246,6 @@ public class ApplicationProperties {
* and paused runs are kept regardless of age.
*/
private int runExpiryMinutes = 30;
/**
* Whether a policy S3 source's custom endpoint may resolve to a loopback, link-local, or
* private address. Off by default so a user-supplied endpoint cannot be pointed at internal
* services (e.g. the cloud metadata address); enable for a self-hosted MinIO or other
* in-network object store.
*/
private boolean allowPrivateS3Endpoints = false;
/** Max inbound webhook body (bytes); over it is rejected 413. Default 100 MB. */
private long webhookMaxBytes = 104857600L;
}
@Data
@@ -60,54 +60,40 @@ public class Provider {
}
private UsernameAttribute validateUsernameAttribute(UsernameAttribute usernameAttribute) {
switch (name) {
case "google" -> {
return validateGoogleUsernameAttribute(usernameAttribute);
}
case "github" -> {
return validateGitHubUsernameAttribute(usernameAttribute);
}
case "keycloak" -> {
return validateKeycloakUsernameAttribute(usernameAttribute);
}
default -> {
return usernameAttribute;
}
}
return switch (name) {
case "google" -> validateGoogleUsernameAttribute(usernameAttribute);
case "github" -> validateGitHubUsernameAttribute(usernameAttribute);
case "keycloak" -> validateKeycloakUsernameAttribute(usernameAttribute);
default -> usernameAttribute;
};
}
private UsernameAttribute validateKeycloakUsernameAttribute(
UsernameAttribute usernameAttribute) {
switch (usernameAttribute) {
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME, PREFERRED_USERNAME -> {
return usernameAttribute;
}
return switch (usernameAttribute) {
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME, PREFERRED_USERNAME -> usernameAttribute;
default ->
throw new UnsupportedClaimException(
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
}
};
}
private UsernameAttribute validateGoogleUsernameAttribute(UsernameAttribute usernameAttribute) {
switch (usernameAttribute) {
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME -> {
return usernameAttribute;
}
return switch (usernameAttribute) {
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME -> usernameAttribute;
default ->
throw new UnsupportedClaimException(
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
}
};
}
private UsernameAttribute validateGitHubUsernameAttribute(UsernameAttribute usernameAttribute) {
switch (usernameAttribute) {
case LOGIN, EMAIL, NAME -> {
return usernameAttribute;
}
return switch (usernameAttribute) {
case LOGIN, EMAIL, NAME -> usernameAttribute;
default ->
throw new UnsupportedClaimException(
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
}
};
}
@Override
@@ -361,8 +361,8 @@ public class PdfMarkdownConverter {
if (xs.isEmpty()) {
return List.of(lines);
}
float minX = xs.get(0);
float maxX = xs.get(xs.size() - 1);
float minX = xs.getFirst();
float maxX = xs.getLast();
float splitAt = (minX + maxX) / 2f;
float biggestGap = 0;
for (int i = 1; i < xs.size(); i++) {
@@ -492,7 +492,7 @@ public class PdfMarkdownConverter {
List<List<Line>> anchorGroups = new ArrayList<>();
List<Line> current = new ArrayList<>();
current.add(cands.get(0));
current.add(cands.getFirst());
for (int i = 1; i < cands.size(); i++) {
float gap = cands.get(i - 1).y - cands.get(i).y;
if (gap > splitThreshold) {
@@ -513,8 +513,8 @@ public class PdfMarkdownConverter {
if (anchors.size() < 2) {
continue;
}
float top = anchors.get(0).y;
float bottom = anchors.get(anchors.size() - 1).y;
float top = anchors.getFirst().y;
float bottom = anchors.getLast().y;
// Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the
// run's vertical span, with a little slack below the last row) into the anchor above.
@@ -674,8 +674,8 @@ public class PdfMarkdownConverter {
float minGutter = Math.max(10f, charWidth * 2.5f);
List<float[]> merged = new ArrayList<>();
for (float[] band : columns) {
if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) {
merged.get(merged.size() - 1)[1] = band[1];
if (!merged.isEmpty() && band[0] - merged.getLast()[1] < minGutter) {
merged.getLast()[1] = band[1];
} else {
merged.add(new float[] {band[0], band[1]});
}
@@ -734,7 +734,7 @@ public class PdfMarkdownConverter {
}
}
StringBuilder sb = new StringBuilder();
sb.append(buildGfmRow(rows.get(0), widths, cols)).append('\n');
sb.append(buildGfmRow(rows.getFirst(), widths, cols)).append('\n');
sb.append('|');
for (int c = 0; c < cols; c++) {
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
@@ -910,8 +910,8 @@ public class PdfMarkdownConverter {
}
// Only merge a sentence continuation between two text paragraphs, never into/out of a
// table.
if (!(output.get(output.size() - 1) instanceof String last)
|| !(pageItems.get(0) instanceof String first)) {
if (!(output.getLast() instanceof String last)
|| !(pageItems.getFirst() instanceof String first)) {
return;
}
if (!first.isEmpty()
@@ -932,13 +932,13 @@ public class PdfMarkdownConverter {
for (Object e : elements) {
if (e instanceof TableBlock tb
&& !out.isEmpty()
&& out.get(out.size() - 1) instanceof TableBlock prev
&& out.getLast() instanceof TableBlock prev
&& columnsMatch(flatten(prev.rows()), flatten(tb.rows()))) {
List<List<Line>> merged = new ArrayList<>(prev.rows());
List<List<Line>> tail = tb.rows();
if (!tail.isEmpty()
&& !prev.rows().isEmpty()
&& rowText(tail.get(0)).equals(rowText(prev.rows().get(0)))) {
&& rowText(tail.getFirst()).equals(rowText(prev.rows().getFirst()))) {
tail = tail.subList(1, tail.size());
}
merged.addAll(tail);
@@ -971,7 +971,7 @@ public class PdfMarkdownConverter {
continue;
}
if (e instanceof TableBlock tb && !tb.rows().isEmpty()) {
return rowText(tb.rows().get(0));
return rowText(tb.rows().getFirst());
}
return null;
}
@@ -1,56 +0,0 @@
package stirling.software.common.service;
/**
* Thread-scoped correlation id for one automation run — a single pipeline, policy, or AI-workflow
* execution over its input file(s).
*
* <p>Automations dispatch each tool step as a separate internal loopback POST via {@link
* InternalApiClient}. The orchestrator opens a run scope around its dispatch loop; {@code
* InternalApiClient} reads {@link #current()} and stamps it on every sub-step request as {@link
* #RUN_ID_HEADER}. The SaaS PAYG interceptor uses that header so all sub-steps of ONE run group
* into a single charge, while two <em>separate</em> runs that happen to touch identical bytes stay
* distinct charges (the old content+time-window grouping merged them).
*
* <p>Sub-steps dispatch synchronously on the orchestrator's own thread (loopback {@code
* RestTemplate}), so this ThreadLocal is visible to {@code InternalApiClient}. The id then crosses
* to the receiving request thread via the HTTP header — never via this ThreadLocal.
*
* <p>No-op when the id is absent (a standalone tool call): the interceptor treats a missing run id
* as "its own charge", which is exactly what a one-off call should be.
*/
public final class AutomationRunContext {
/** Header carrying the run id on internal sub-step dispatches. */
public static final String RUN_ID_HEADER = "X-Stirling-Run-Id";
private static final ThreadLocal<String> CURRENT = new ThreadLocal<>();
private AutomationRunContext() {}
/**
* Opens a run scope on the current thread. Returns an {@link AutoCloseable} that restores the
* previously-active id (nesting-safe) — use in try-with-resources around the dispatch loop.
*/
public static Scope open(String runId) {
String previous = CURRENT.get();
CURRENT.set(runId);
return () -> {
if (previous == null) {
CURRENT.remove();
} else {
CURRENT.set(previous);
}
};
}
/** The run id active on this thread, or {@code null} when not inside a run scope. */
public static String current() {
return CURRENT.get();
}
/** AutoCloseable whose {@link #close()} declares no checked exception. */
public interface Scope extends AutoCloseable {
@Override
void close();
}
}
@@ -8,7 +8,6 @@ import java.nio.file.Files;
import java.time.Duration;
import java.util.regex.Pattern;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.core.io.FileSystemResource;
@@ -61,17 +60,6 @@ public class InternalApiClient {
*/
public static final String AUTOMATION_HEADER = "X-Stirling-Automation";
/**
* Header carrying the parent policy's name onto each sub-step dispatch, read from MDC key
* {@link #POLICY_NAME_MDC_KEY} (set by the policy runner on the worker thread). Lets the audit
* layer attribute a tool step to the policy that ran it, instead of showing it as a bare direct
* call.
*/
public static final String POLICY_NAME_HEADER = "X-Stirling-Policy-Name";
/** MDC key the policy runner stamps with the running policy's name; forwarded as a header. */
public static final String POLICY_NAME_MDC_KEY = "auditPolicyName";
private final ServletContext servletContext;
private final UserServiceInterface userService;
private final TempFileManager tempFileManager;
@@ -123,27 +111,6 @@ public class InternalApiClient {
// step inside a policy run must bill as AUTOMATION, not AI). Set unconditionally because
// every caller of this dispatcher is an automation surface by design.
headers.add(AUTOMATION_HEADER, "true");
// Propagate the current automation run id (set by the orchestrator around its dispatch
// loop) so the PAYG interceptor groups every sub-step of this one run into a single charge,
// and never merges two separate runs that happen to touch identical bytes. Absent → the
// receiving call is treated as standalone. See AutomationRunContext.
String runId = AutomationRunContext.current();
if (runId != null && !runId.isEmpty()) {
headers.add(AutomationRunContext.RUN_ID_HEADER, runId);
}
// Forward the parent policy name (set in MDC by the policy runner) so the audited sub-step
// ties back to its policy. Single-line, length-capped: it becomes an HTTP header value.
String policyName = MDC.get(POLICY_NAME_MDC_KEY);
if (policyName != null && !policyName.isBlank()) {
String safe = policyName.replaceAll("[\\r\\n]", " ").trim();
if (safe.length() > 200) {
safe = safe.substring(0, 200);
}
if (!safe.isEmpty()) {
headers.add(POLICY_NAME_HEADER, safe);
}
}
// A no-file ai/tools call (e.g. create-pdf-from-html-agent) sends only string params, so
// without this RestTemplate would use urlencoded instead of the multipart the controller
@@ -152,7 +119,7 @@ public class InternalApiClient {
boolean hasFilePart =
body.values().stream()
.flatMap(java.util.List::stream)
.anyMatch(v -> v instanceof Resource);
.anyMatch(Resource.class::isInstance);
if (isAiTool && !hasFilePart) {
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
}
@@ -144,10 +144,8 @@ public class TempFileCleanupService {
int directoriesDeletedCount = 0;
for (Path directory : registry.getTempDirectories()) {
try {
if (Files.exists(directory)
&& shouldDeleteRegisteredDirectory(directory, maxAgeMillis)) {
if (Files.exists(directory)) {
GeneralUtils.deleteDirectory(directory);
registry.unregisterDirectory(directory);
directoriesDeletedCount++;
log.debug("Cleaned up temporary directory: {}", directory);
}
@@ -277,21 +275,6 @@ public class TempFileCleanupService {
return totalDeletedCount.get();
}
private boolean shouldDeleteRegisteredDirectory(Path directory, long maxAgeMillis) {
if (maxAgeMillis <= 0) {
return true;
}
try {
long currentTime = System.currentTimeMillis();
long lastModified = Files.getLastModifiedTime(directory).toMillis();
return (currentTime - lastModified) > maxAgeMillis;
} catch (IOException e) {
log.debug("Could not check directory age, skipping cleanup: {}", directory, e);
return false;
}
}
/** Get the system temp directory path based on configuration or system property. */
private Path getSystemTempPath() {
String systemTempDir =
@@ -140,20 +140,25 @@ public class ChecksumUtils {
for (String algorithm : algorithms) {
String key = algorithm; // keep original key for output
switch (algorithm.toUpperCase(Locale.ROOT)) {
case "CRC32":
checksums.put(key, new CRC32());
break;
case "ADLER32":
checksums.put(key, new Adler32());
break;
default:
try {
// For MessageDigest, pass the original name (case-insensitive per JCA)
digests.put(key, MessageDigest.getInstance(algorithm));
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("Unsupported algorithm: " + algorithm, e);
}
Object digestOrChecksum =
switch (algorithm.toUpperCase(Locale.ROOT)) {
case "CRC32" -> new CRC32();
case "ADLER32" -> new Adler32();
default -> {
try {
// For MessageDigest, pass the original name (case-insensitive
// per JCA)
yield MessageDigest.getInstance(algorithm);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(
"Unsupported algorithm: " + algorithm, e);
}
}
};
if (digestOrChecksum instanceof Checksum checksum) {
checksums.put(key, checksum);
} else {
digests.put(key, (MessageDigest) digestOrChecksum);
}
}
@@ -751,7 +751,7 @@ public class ExceptionUtils {
String targetDescription;
if (errorInfo.affectedPages() != null && !errorInfo.affectedPages().isEmpty()) {
if (errorInfo.affectedPages().size() == 1) {
targetDescription = "page " + errorInfo.affectedPages().get(0);
targetDescription = "page " + errorInfo.affectedPages().getFirst();
} else {
targetDescription =
"pages "
@@ -848,7 +848,7 @@ public class ExceptionUtils {
}
// Use the first page number, or null if none found
Integer pageNumber = affectedPages.isEmpty() ? null : affectedPages.get(0);
Integer pageNumber = affectedPages.isEmpty() ? null : affectedPages.getFirst();
return new GhostscriptErrorInfo(
ErrorCode.GHOSTSCRIPT_PAGE_DRAWING,
@@ -114,7 +114,7 @@ public enum FormFieldTypeSupport {
return;
}
PDAnnotationWidget widget = checkBox.getWidgets().get(0);
PDAnnotationWidget widget = checkBox.getWidgets().getFirst();
PDAppearanceCharacteristicsDictionary appearanceChars =
widget.getAppearanceCharacteristics();
@@ -88,28 +88,16 @@ public class FormUtils {
* text)
*/
public String detectFieldType(PDField field) {
if (field instanceof PDSignatureField) {
return FIELD_TYPE_SIGNATURE;
}
if (field instanceof PDPushButton) {
return FIELD_TYPE_BUTTON;
}
if (field instanceof PDTextField) {
return FIELD_TYPE_TEXT;
}
if (field instanceof PDCheckBox) {
return FIELD_TYPE_CHECKBOX;
}
if (field instanceof PDComboBox) {
return FIELD_TYPE_COMBOBOX;
}
if (field instanceof PDListBox) {
return FIELD_TYPE_LISTBOX;
}
if (field instanceof PDRadioButton) {
return FIELD_TYPE_RADIO;
}
return FIELD_TYPE_TEXT;
return switch (field) {
case PDSignatureField ignored -> FIELD_TYPE_SIGNATURE;
case PDPushButton ignored -> FIELD_TYPE_BUTTON;
case PDTextField ignored -> FIELD_TYPE_TEXT;
case PDCheckBox ignored -> FIELD_TYPE_CHECKBOX;
case PDComboBox ignored -> FIELD_TYPE_COMBOBOX;
case PDListBox ignored -> FIELD_TYPE_LISTBOX;
case PDRadioButton ignored -> FIELD_TYPE_RADIO;
case null, default -> FIELD_TYPE_TEXT;
};
}
public List<FormFieldInfo> extractFormFields(PDDocument document) {
@@ -583,22 +571,17 @@ public class FormUtils {
continue;
}
String type = info.type();
Object value;
switch (type) {
case FIELD_TYPE_CHECKBOX:
value = isChecked(info.value()) ? Boolean.TRUE : Boolean.FALSE;
break;
case FIELD_TYPE_LISTBOX:
if (info.multiSelect()) {
value = new ArrayList<>();
} else {
value = safeDefault(info.value());
}
break;
case FIELD_TYPE_BUTTON, FIELD_TYPE_SIGNATURE:
continue; // skip non-fillable
default:
value = safeDefault(info.value());
Object value =
switch (type) {
case FIELD_TYPE_CHECKBOX ->
isChecked(info.value()) ? Boolean.TRUE : Boolean.FALSE;
case FIELD_TYPE_LISTBOX ->
info.multiSelect() ? new ArrayList<>() : safeDefault(info.value());
case FIELD_TYPE_BUTTON, FIELD_TYPE_SIGNATURE -> null;
default -> safeDefault(info.value());
};
if (value == null) {
continue; // skip non-fillable
}
record.put(info.name(), value);
}
@@ -949,44 +932,44 @@ public class FormUtils {
if (selection == null || selection.trim().isEmpty()) return null;
List<String> filtered =
filterChoiceSelections(List.of(selection), allowedOptions, fieldName);
return filtered.isEmpty() ? null : filtered.get(0);
return filtered.isEmpty() ? null : filtered.getFirst();
}
private void applyValueToField(PDField field, String value, boolean strict) throws IOException {
try {
if (field instanceof PDTextField textField) {
setTextValue(textField, value);
} else if (field instanceof PDCheckBox checkBox) {
LinkedHashSet<String> candidateStates = collectCheckBoxStates(checkBox);
boolean shouldCheck = shouldCheckBoxBeChecked(value, candidateStates);
try {
if (shouldCheck) {
checkBox.check();
} else {
checkBox.unCheck();
}
} catch (IOException checkProblem) {
log.warn(
"Failed to set checkbox state for '{}': {}",
field.getFullyQualifiedName(),
checkProblem.getMessage(),
checkProblem);
if (strict) {
throw checkProblem;
switch (field) {
case PDTextField textField -> setTextValue(textField, value);
case PDCheckBox checkBox -> {
LinkedHashSet<String> candidateStates = collectCheckBoxStates(checkBox);
boolean shouldCheck = shouldCheckBoxBeChecked(value, candidateStates);
try {
if (shouldCheck) {
checkBox.check();
} else {
checkBox.unCheck();
}
} catch (IOException checkProblem) {
log.warn(
"Failed to set checkbox state for '{}': {}",
field.getFullyQualifiedName(),
checkProblem.getMessage(),
checkProblem);
if (strict) {
throw checkProblem;
}
}
}
} else if (field instanceof PDRadioButton radioButton) {
if (value != null && !value.isBlank()) {
radioButton.setValue(value);
case PDRadioButton radioButton -> {
if (value != null && !value.isBlank()) {
radioButton.setValue(value);
}
}
} else if (field instanceof PDChoice choiceField) {
applyChoiceValue(choiceField, value);
} else if (field instanceof PDPushButton) {
log.debug("Ignore Push button");
} else if (field instanceof PDSignatureField) {
log.debug("Skipping signature field '{}'", field.getFullyQualifiedName());
} else {
field.setValue(value != null ? value : "");
case PDChoice choiceField -> applyChoiceValue(choiceField, value);
case PDPushButton ignored -> log.debug("Ignore Push button");
case PDSignatureField ignored ->
log.debug("Skipping signature field '{}'", field.getFullyQualifiedName());
case null -> log.warn("Attempted to set value on null field");
default -> field.setValue(value != null ? value : "");
}
} catch (Exception e) {
log.warn(
@@ -1306,37 +1289,42 @@ public class FormUtils {
List<String> resolveOptions(PDTerminalField field) {
try {
if (field instanceof PDChoice choice) {
LinkedHashSet<String> allowed = new LinkedHashSet<>();
List<String> exportValues = choice.getOptionsExportValues();
List<String> displayValues = choice.getOptionsDisplayValues();
return switch (field) {
case PDChoice choice -> {
LinkedHashSet<String> allowed = new LinkedHashSet<>();
List<String> exportValues = choice.getOptionsExportValues();
List<String> displayValues = choice.getOptionsDisplayValues();
if (exportValues != null) {
exportValues.stream()
.filter(Objects::nonNull)
.map(String::trim)
.filter(s -> !s.isEmpty())
.forEach(allowed::add);
if (exportValues != null) {
exportValues.stream()
.filter(Objects::nonNull)
.map(String::trim)
.filter(s -> !s.isEmpty())
.forEach(allowed::add);
}
if (displayValues != null) {
displayValues.stream()
.filter(Objects::nonNull)
.map(String::trim)
.filter(s -> !s.isEmpty())
.forEach(allowed::add);
}
yield new ArrayList<>(allowed);
}
if (displayValues != null) {
displayValues.stream()
.filter(Objects::nonNull)
.map(String::trim)
.filter(s -> !s.isEmpty())
.forEach(allowed::add);
case PDRadioButton radio -> {
List<String> exports = radio.getExportValues();
yield exports != null && !exports.isEmpty()
? new ArrayList<>(exports)
: Collections.emptyList();
}
return new ArrayList<>(allowed);
} else if (field instanceof PDRadioButton radio) {
List<String> exports = radio.getExportValues();
if (exports != null && !exports.isEmpty()) {
return new ArrayList<>(exports);
case PDCheckBox checkBox -> {
List<String> exports = checkBox.getExportValues();
yield exports != null && !exports.isEmpty()
? new ArrayList<>(exports)
: Collections.emptyList();
}
} else if (field instanceof PDCheckBox checkBox) {
List<String> exports = checkBox.getExportValues();
if (exports != null && !exports.isEmpty()) {
return new ArrayList<>(exports);
}
}
case null, default -> Collections.emptyList();
};
} catch (Exception e) {
log.debug(
"Failed to resolve options for field '{}': {}",
@@ -1465,7 +1453,7 @@ public class FormUtils {
// Only check options for choice-type fields (combobox, listbox, radio)
if (CHOICE_FIELD_TYPES.contains(type) && options != null && !options.isEmpty()) {
String optionCandidate = cleanLabel(options.get(0));
String optionCandidate = cleanLabel(options.getFirst());
if (optionCandidate != null && !looksGeneric(optionCandidate)) {
return optionCandidate;
}
@@ -1557,7 +1545,7 @@ public class FormUtils {
continue;
}
PDAnnotationWidget widget = widgets.get(0);
PDAnnotationWidget widget = widgets.getFirst();
PDRectangle originalRectangle = cloneRectangle(widget.getRectangle());
PDPage page = resolveWidgetPage(document, widget, null);
if (page == null || originalRectangle == null) {
@@ -2446,19 +2434,19 @@ public class FormUtils {
private static int firstWidgetPageIndex(FormFieldWithCoordinates f) {
return (f.getWidgets() != null && !f.getWidgets().isEmpty())
? f.getWidgets().get(0).getPageIndex()
? f.getWidgets().getFirst().getPageIndex()
: -1;
}
private static float firstWidgetY(FormFieldWithCoordinates f) {
return (f.getWidgets() != null && !f.getWidgets().isEmpty())
? f.getWidgets().get(0).getY()
? f.getWidgets().getFirst().getY()
: 0;
}
private static float firstWidgetX(FormFieldWithCoordinates f) {
return (f.getWidgets() != null && !f.getWidgets().isEmpty())
? f.getWidgets().get(0).getX()
? f.getWidgets().getFirst().getX()
: 0;
}
@@ -26,29 +26,27 @@ import lombok.extern.slf4j.Slf4j;
public class ImageProcessingUtils {
static BufferedImage convertColorType(BufferedImage sourceImage, String colorType) {
BufferedImage convertedImage;
switch (colorType) {
case "greyscale":
convertedImage =
return switch (colorType) {
case "greyscale" -> {
BufferedImage convertedImage =
new BufferedImage(
sourceImage.getWidth(),
sourceImage.getHeight(),
BufferedImage.TYPE_BYTE_GRAY);
convertedImage.getGraphics().drawImage(sourceImage, 0, 0, null);
break;
case "blackwhite":
convertedImage =
yield convertedImage;
}
case "blackwhite" -> {
BufferedImage convertedImage =
new BufferedImage(
sourceImage.getWidth(),
sourceImage.getHeight(),
BufferedImage.TYPE_BYTE_BINARY);
convertedImage.getGraphics().drawImage(sourceImage, 0, 0, null);
break;
default: // full color
convertedImage = sourceImage;
break;
}
return convertedImage;
yield convertedImage;
}
default -> sourceImage;
};
}
public static byte[] getImageData(BufferedImage image) {
@@ -330,7 +330,7 @@ public class PDFToFile {
if (outputFiles.size() == 1) {
// Return single output file
File outputFile = outputFiles.get(0);
File outputFile = outputFiles.getFirst();
if ("txt:Text".equals(outputFormat)) {
outputFormat = "txt";
}
@@ -307,7 +307,7 @@ public class ProcessExecutor {
boolean isQpdf =
commandToRun != null
&& !commandToRun.isEmpty()
&& commandToRun.get(0).contains("qpdf");
&& commandToRun.getFirst().contains("qpdf");
if (!outputLines.isEmpty()) {
String outputMessage = String.join("\n", outputLines);
@@ -370,7 +370,7 @@ public class ProcessExecutor {
}
// Check if this is a UNO conversion by looking for unoconvert executable
String executable = command.get(0);
String executable = command.getFirst();
if (executable != null) {
// Extract basename from path for matching
String basename = executable;
@@ -504,7 +504,7 @@ public class ProcessExecutor {
}
// Validate executable (first argument)
String executable = command.get(0);
String executable = command.getFirst();
if (executable == null || executable.isBlank()) {
throw new IllegalArgumentException("Command executable must not be empty");
}
@@ -198,8 +198,6 @@ public class RequestUriUtils {
|| trimmedUri.startsWith("/readiness")
|| trimmedUri.startsWith(
"/api/v1/mobile-scanner/") // Mobile scanner endpoints (no auth)
// Webhook receiver: authenticated per-request by HMAC signature, not a session.
|| trimmedUri.startsWith("/api/v1/webhooks/")
|| trimmedUri.startsWith("/v1/api-docs")
// Workflow participant endpoints - access controlled by share tokens, not login
|| trimmedUri.startsWith("/api/v1/workflow/participant/")
@@ -155,7 +155,6 @@ public class TempFileManager {
if (directory != null && Files.isDirectory(directory)) {
try {
GeneralUtils.deleteDirectory(directory);
registry.unregisterDirectory(directory);
log.debug("Deleted temp directory: {}", directory.toString());
} catch (IOException e) {
log.warn("Failed to delete temp directory: {}", directory.toString(), e);
@@ -85,18 +85,6 @@ public class TempFileRegistry {
return directory;
}
/**
* Unregister a temporary directory from the registry.
*
* @param directory The directory to unregister
*/
public void unregisterDirectory(Path directory) {
if (directory != null) {
tempDirectories.remove(directory);
log.debug("Unregistered temp directory: {}", directory.toString());
}
}
/**
* Register a third-party temporary file that requires special handling.
*
@@ -114,7 +114,7 @@ public class YamlHelper {
for (NodeTuple tuple : mappingNode.getValue()) {
ScalarNode keyNode = (tuple.getKeyNode() instanceof ScalarNode sk) ? sk : null;
if (keyNode == null || !keyNode.getValue().equals(keys.get(0))) {
if (keyNode == null || !keyNode.getValue().equals(keys.getFirst())) {
updatedTuples.add(tuple);
continue;
}
@@ -176,13 +176,11 @@ class TempFileCleanupServiceMoreTest {
class ScheduledCleanup {
@Test
@DisplayName("deletes stale registered temp directories and reports counts")
@DisplayName("deletes registered temp directories and reports counts")
void deletesRegisteredDirectories() throws IOException {
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(2);
Path regDir = Files.createDirectories(tempDir.resolve("registeredDir"));
Files.createFile(regDir.resolve("inside.txt"));
Files.setLastModifiedTime(
regDir, FileTime.fromMillis(System.currentTimeMillis() - 2L * 60 * 60 * 1000));
Set<Path> dirs = new HashSet<>();
dirs.add(regDir);
when(registry.getTempDirectories()).thenReturn(dirs);
@@ -195,22 +193,6 @@ class TempFileCleanupServiceMoreTest {
verify(tempFileManager).cleanupOldTempFiles(anyLong());
}
@Test
@DisplayName("keeps a fresh registered temp directory")
void keepsFreshRegisteredDirectory() throws IOException {
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(0);
Path regDir = Files.createDirectories(tempDir.resolve("freshRegisteredDir"));
Files.createFile(regDir.resolve("inside.txt"));
Set<Path> dirs = new HashSet<>();
dirs.add(regDir);
when(registry.getTempDirectories()).thenReturn(dirs);
lenient().when(registry.contains(any(File.class))).thenReturn(false);
withIsolatedUserHome(cleanupService::scheduledCleanup);
assertThat(Files.exists(regDir)).isTrue();
}
@Test
@DisplayName("skips a registered directory that no longer exists")
void skipsMissingRegisteredDirectory() {
@@ -721,7 +721,7 @@ class PDFToFileTest {
.thenAnswer(
invocation -> {
List<String> args = invocation.getArgument(0);
String outputPath = args.get(args.size() - 1);
String outputPath = args.getLast();
Files.write(Path.of(outputPath), "Fake DOCX content".getBytes());
return mockExecutorResult;
});
@@ -176,13 +176,6 @@ class RequestUriUtilsTest {
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/api/v1/convert", ""));
}
@Test
void testIsPublicAuthEndpoint_webhookReceiver() {
// The webhook source receiver authenticates each delivery by HMAC signature, not a session.
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/api/v1/webhooks/whk_abc123", ""));
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/app/api/v1/webhooks/whk_abc123", "/app"));
}
@Test
void testIsPublicAuthEndpoint_withContextPath() {
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/app/login", "/app"));
@@ -1,6 +1,5 @@
package stirling.software.SPDF.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.servlet.MultipartConfigFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -22,7 +21,11 @@ import stirling.software.SPDF.controller.web.UploadLimitService;
@Slf4j
public class MultipartConfiguration {
@Autowired private UploadLimitService uploadLimitService;
private final UploadLimitService uploadLimitService;
public MultipartConfiguration(UploadLimitService uploadLimitService) {
this.uploadLimitService = uploadLimitService;
}
/**
* Creates MultipartConfigElement that respects fileUploadLimit from settings.yml or environment
@@ -294,24 +294,25 @@ public class BookletImpositionController {
// Apply rotation if needed (rotate about origin), then translate to keep in cell
switch (rot) {
case 90:
case 90 -> {
cs.transform(Matrix.getRotateInstance(Math.PI / 2, 0, 0));
// After 90° CCW, the content spans x in [-r.getHeight(), 0] and y in [0,
// r.getWidth()]
cs.transform(Matrix.getTranslateInstance(0, -r.getWidth()));
break;
case 180:
}
case 180 -> {
cs.transform(Matrix.getRotateInstance(Math.PI, 0, 0));
cs.transform(Matrix.getTranslateInstance(-r.getWidth(), -r.getHeight()));
break;
case 270:
}
case 270 -> {
cs.transform(Matrix.getRotateInstance(3 * Math.PI / 2, 0, 0));
// After 270° CCW, the content spans x in [0, r.getHeight()] and y in
// [-r.getWidth(), 0]
cs.transform(Matrix.getTranslateInstance(-r.getHeight(), 0));
break;
default:
}
default -> {
// 0°: no-op
}
}
// Reuse LayerUtility passed from caller
@@ -9,7 +9,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
@@ -58,8 +57,6 @@ import stirling.software.jpdfium.doc.PdfBookmarkEditor.BookmarkTree;
@Slf4j
@RequiredArgsConstructor
public class MergeController {
private static final Pattern QUOTE_WRAP_PATTERN = Pattern.compile("^\"|\"$");
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final TempFileManager tempFileManager;
@@ -161,30 +158,6 @@ public class MergeController {
};
}
private String[] parseClientFileIds(String clientFileIds) {
if (clientFileIds == null || clientFileIds.trim().isEmpty()) {
return new String[0];
}
try {
String trimmed = clientFileIds.trim();
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
String inside = trimmed.substring(1, trimmed.length() - 1).trim();
if (inside.isEmpty()) {
return new String[0];
}
String[] parts = inside.split(",");
String[] result = new String[parts.length];
for (int i = 0; i < parts.length; i++) {
result[i] = QUOTE_WRAP_PATTERN.matcher(parts[i].trim()).replaceAll("");
}
return result;
}
} catch (Exception e) {
log.warn("Failed to parse client file IDs: {}", clientFileIds, e);
}
return new String[0];
}
private void addTableOfContents(PDDocument mergedDocument, MultipartFile[] files) {
PDDocumentOutline outline = new PDDocumentOutline();
mergedDocument.getDocumentCatalog().setDocumentOutline(outline);
@@ -125,17 +125,14 @@ public class UIDataController {
pipelineConfigs.add(content);
}
for (String config : pipelineConfigs) {
for (int i = 0; i < pipelineConfigs.size(); i++) {
String config = pipelineConfigs.get(i);
Map<String, Object> jsonContent =
objectMapper.readValue(
config, new TypeReference<Map<String, Object>>() {});
String name = (String) jsonContent.get("name");
if (name == null || name.length() < 1) {
String filename =
jsonFiles
.get(pipelineConfigs.indexOf(config))
.getFileName()
.toString();
String filename = jsonFiles.get(i).getFileName().toString();
name = filename.substring(0, filename.lastIndexOf('.'));
}
Map<String, String> configWithName = new HashMap<>();
@@ -301,20 +298,14 @@ public class UIDataController {
}
private static String getFormatFromExtension(String extension) {
switch (extension) {
case "ttf":
return "truetype";
case "woff":
return "woff";
case "woff2":
return "woff2";
case "eot":
return "embedded-opentype";
case "svg":
return "svg";
default:
return "";
}
return switch (extension) {
case "ttf" -> "truetype";
case "woff" -> "woff";
case "woff2" -> "woff2";
case "eot" -> "embedded-opentype";
case "svg" -> "svg";
default -> "";
};
}
}
}
@@ -187,7 +187,7 @@ public class ConvertImgPDFController {
}
if (webpFiles.size() == 1) {
Path webpFilePath = webpFiles.get(0);
Path webpFilePath = webpFiles.getFirst();
byte[] webpBytes = Files.readAllBytes(webpFilePath);
Files.deleteIfExists(tempFile);
tempFile = null;
@@ -147,7 +147,7 @@ public class ConvertSvgToPDF {
String outputFilename =
filenames.isEmpty()
? "combined_svgs.pdf"
: GeneralUtils.generateFilename(filenames.get(0), "_combined.pdf");
: GeneralUtils.generateFilename(filenames.getFirst(), "_combined.pdf");
log.info("Successfully combined {} SVGs into single PDF", sanitizedSvgs.size());
@@ -203,7 +203,7 @@ public class ConvertSvgToPDF {
try {
if (convertedPdfs.size() == 1) {
ConvertedPdf pdf = convertedPdfs.get(0);
ConvertedPdf pdf = convertedPdfs.getFirst();
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
try {
Files.write(tempOut.getPath(), pdf.content);
@@ -218,7 +218,7 @@ public class ConvertSvgToPDF {
filenames.isEmpty()
? "converted_svgs.zip"
: GeneralUtils.generateFilename(
filenames.get(0), "_converted_svgs.zip");
filenames.getFirst(), "_converted_svgs.zip");
TempFile zipFile = createZipFromPdfs(convertedPdfs);
return WebResponseUtils.zipFileToWebResponse(zipFile, zipFilename);
} catch (IOException e) {
@@ -82,7 +82,7 @@ public class ExtractCSVController {
if (csvEntries.isEmpty()) {
return ResponseEntity.noContent().build();
} else if (csvEntries.size() == 1) {
return createCsvResponse(csvEntries.get(0), baseName);
return createCsvResponse(csvEntries.getFirst(), baseName);
} else {
return createZipResponse(csvEntries, baseName);
}
@@ -113,7 +113,9 @@ public class AutoRenameController {
mergedLineInfos.sort(
Comparator.comparing((LineInfo li) -> li.fontSize).reversed());
String title =
mergedLineInfos.isEmpty() ? null : mergedLineInfos.get(0).text;
mergedLineInfos.isEmpty()
? null
: mergedLineInfos.getFirst().text;
return title != null
? title
@@ -333,7 +333,7 @@ public class AutoSplitPdfController {
}
if (!splitDocuments.isEmpty() && !isValidQrCode) {
splitDocuments.get(splitDocuments.size() - 1).addPage(document.getPage(page));
splitDocuments.getLast().addPage(document.getPage(page));
} else if (page == 0) {
PDDocument firstDocument = new PDDocument();
firstDocument.addPage(document.getPage(page));
@@ -267,7 +267,7 @@ public class CompressController {
if (references.isEmpty()) continue;
// Get the first instance of this image
PDImageXObject originalImage = getOriginalImage(doc, references.get(0));
PDImageXObject originalImage = getOriginalImage(doc, references.getFirst());
// Track original size
int originalSize = (int) originalImage.getCOSObject().getLength();
@@ -1167,7 +1167,7 @@ public class CompressController {
List<ImageReference> references = entry.getValue();
if (references.isEmpty()) continue;
PDImageXObject originalImage = getOriginalImage(doc, references.get(0));
PDImageXObject originalImage = getOriginalImage(doc, references.getFirst());
int originalSize = (int) originalImage.getCOSObject().getLength();
stats.totalOriginalBytes += originalSize;
@@ -211,7 +211,7 @@ public class ExtractImageScansController {
} else {
// Return the processed image as a response
byte[] imageBytes = processedImageBytes.get(0);
byte[] imageBytes = processedImageBytes.getFirst();
finalOutput = tempFileManager.createManagedTempFile(".png");
try (OutputStream out = Files.newOutputStream(finalOutput.getPath())) {
out.write(imageBytes);
@@ -10,7 +10,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.UUID;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
@@ -28,7 +27,6 @@ import stirling.software.SPDF.model.PipelineConfig;
import stirling.software.SPDF.model.PipelineOperation;
import stirling.software.SPDF.model.PipelineResult;
import stirling.software.SPDF.service.ApiDocService;
import stirling.software.common.service.AutomationRunContext;
import stirling.software.common.service.InternalApiClient;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.ZipExtractionUtils;
@@ -73,17 +71,6 @@ public class PipelineProcessor {
PipelineResult runPipelineAgainstFiles(List<Resource> outputFiles, PipelineConfig config)
throws Exception {
// One pipeline execution = one automation run. Scope a run id so every tool sub-step
// dispatched via InternalApiClient groups into a single charge on the SaaS billing side
// (see AutomationRunContext); pipeline steps run synchronously on this thread.
try (AutomationRunContext.Scope ignored =
AutomationRunContext.open(UUID.randomUUID().toString())) {
return runPipelineAgainstFilesInternal(outputFiles, config);
}
}
private PipelineResult runPipelineAgainstFilesInternal(
List<Resource> outputFiles, PipelineConfig config) throws Exception {
PipelineResult result = new PipelineResult();
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
@@ -404,7 +404,7 @@ public class CertSignController {
PDAcroForm acroForm = new PDAcroForm(doc);
doc.getDocumentCatalog().setAcroForm(acroForm);
PDSignatureField signatureField = new PDSignatureField(acroForm);
PDAnnotationWidget widget = signatureField.getWidgets().get(0);
PDAnnotationWidget widget = signatureField.getWidgets().getFirst();
List<PDField> acroFormFields = acroForm.getFields();
acroForm.setSignaturesExist(true);
acroForm.setAppendOnly(true);
@@ -136,17 +136,11 @@ public class RedactController {
+ "Users can provide text patterns to redact, with options for regex and whole word matching. "
+ "Input:PDF Output:PDF Type:SISO")
public ResponseEntity<Resource> redactPdf(@ModelAttribute RedactPdfRequest request) {
String rawListOfText = request.getListOfText();
String[] listOfText = request.getListOfText().split("\n");
boolean useRegex = Boolean.TRUE.equals(request.getUseRegex());
boolean wholeWordSearchBool = Boolean.TRUE.equals(request.getWholeWordSearch());
if (rawListOfText == null || rawListOfText.trim().isEmpty()) {
throw ExceptionUtils.createIllegalArgumentException(
"error.redaction.no.patterns", "No text patterns provided for redaction");
}
String[] listOfText = rawListOfText.split("\n");
if (listOfText.length == 1 && listOfText[0].trim().isEmpty()) {
if (listOfText.length == 0 || (listOfText.length == 1 && listOfText[0].trim().isEmpty())) {
throw ExceptionUtils.createIllegalArgumentException(
"error.redaction.no.patterns", "No text patterns provided for redaction");
}
@@ -634,7 +634,7 @@ class RedactExecuteService {
PageColumnLayout layout =
PageColumnLayout.fromLineBoxes(extractor.getLineBoxes(), pageWidth);
if (layout.columnCount() > 1) {
float[] g = layout.gutters().get(0);
float[] g = layout.gutters().getFirst();
log.info(
"[redact/execute] page {} layout: 2 cols, gutter x=[{}, {}]",
pageIdx + 1,
@@ -60,7 +60,7 @@ public class RemoveCertSignController {
// Remove signature fields safely
List<PDField> fieldsToRemove =
acroForm.getFields().stream()
.filter(field -> field instanceof PDSignatureField)
.filter(PDSignatureField.class::isInstance)
.toList();
if (!fieldsToRemove.isEmpty()) {
@@ -2,7 +2,6 @@ package stirling.software.SPDF.controller.web;
import java.util.Locale;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
@@ -13,7 +12,11 @@ import stirling.software.common.model.ApplicationProperties;
@Slf4j
public class UploadLimitService {
@Autowired private ApplicationProperties applicationProperties;
private final ApplicationProperties applicationProperties;
public UploadLimitService(ApplicationProperties applicationProperties) {
this.applicationProperties = applicationProperties;
}
public long getUploadLimit() {
String raw =
@@ -2688,7 +2688,7 @@ public class PdfJsonConversionService {
// Find which page the field is on
PDAnnotationWidget widget =
field.getWidgets().isEmpty() ? null : field.getWidgets().get(0);
field.getWidgets().isEmpty() ? null : field.getWidgets().getFirst();
if (widget != null) {
PDPage fieldPage = widget.getPage();
if (fieldPage != null) {
@@ -3164,7 +3164,7 @@ public class PdfJsonConversionService {
&& imageObjectNames != null
&& !imageObjectNames.isEmpty()
&& !targetTokens.isEmpty()) {
Object previous = targetTokens.get(targetTokens.size() - 1);
Object previous = targetTokens.getLast();
if (previous instanceof COSName cosName
&& imageObjectNames.contains(cosName.getName())) {
targetTokens.remove(targetTokens.size() - 1);
@@ -5246,7 +5246,7 @@ public class PdfJsonConversionService {
throws IOException {
if (OperatorName.DRAW_OBJECT.equals(operator.getName())
&& !operands.isEmpty()
&& operands.get(0) instanceof COSName name) {
&& operands.getFirst() instanceof COSName name) {
currentXObjectName = name;
}
super.processOperator(operator, operands);
@@ -420,7 +420,7 @@ public class PdfJsonImageService {
throws IOException {
if (OperatorName.DRAW_OBJECT.equals(operator.getName())
&& !operands.isEmpty()
&& operands.get(0) instanceof COSName name) {
&& operands.getFirst() instanceof COSName name) {
currentXObjectName = name;
}
super.processOperator(operator, operands);
@@ -137,7 +137,7 @@ public class JobController {
if (result.hasFiles() && !result.hasMultipleFiles()) {
try {
List<ResultFile> files = result.getAllResultFiles();
ResultFile singleFile = files.get(0);
ResultFile singleFile = files.getFirst();
byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId());
return ResponseEntity.ok()
File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 6.4 KiB

After

Width:  |  Height:  |  Size: 6.4 KiB

@@ -1 +1,18 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 24 24"><symbol id="icon-redact-auto" viewBox="0 0 24 24"><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><rect width="24" height="24" style="fill:none"/><g><path d="M17.541,15.64258a.91793.91793,0,0,1,.55469-.18555h1.1084a.91586.91586,0,0,1,.55469.18555,1.30889,1.30889,0,0,1,.40429.499,1.57206,1.57206,0,0,1,.15039.68457v5.47754H19.2041V20.21094H18.0957v2.09277H16.9873V16.82617a1.55843,1.55843,0,0,1,.15039-.68457A1.2979,1.2979,0,0,1,17.541,15.64258Zm1.66309,1.10547H18.0957v2.17187h1.1084Z" style="fill:currentColor"/><path d="M5.68653,22.30351a2.00588,2.00588,0,0,1-2-2v-16A1.92585,1.92585,0,0,1,4.274,2.891a1.92585,1.92585,0,0,1,1.4125-.5875h8l6,6v5.66931h-2V9.30351h-5v-5h-7v16h9.74021v2Z" style="fill:currentColor"/><rect width="4.338" height=".795" x="7.698" y="10.432" style="fill:currentColor"/><rect width="7.312" height="1.213" x="7.698" y="12.169" style="fill:currentColor"/><rect width="7.312" height="1.213" x="7.698" y="17.146" style="fill:currentColor"/><rect width="7.312" height=".575" x="7.698" y="14.324" style="fill:currentColor"/><rect width="5.256" height=".448" x="7.698" y="15.798" style="fill:currentColor"/></g></g></g></symbol></svg>
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 24 24">
<symbol id="icon-redact-auto" viewBox="0 0 24 24"> <g id="Layer_2" data-name="Layer 2">
<g id="Layer_1-2" data-name="Layer 1">
<rect width="24" height="24" style="fill: none"/>
<g>
<path d="M17.541,15.64258a.91793.91793,0,0,1,.55469-.18555h1.1084a.91586.91586,0,0,1,.55469.18555,1.30889,1.30889,0,0,1,.40429.499,1.57206,1.57206,0,0,1,.15039.68457v5.47754H19.2041V20.21094H18.0957v2.09277H16.9873V16.82617a1.55843,1.55843,0,0,1,.15039-.68457A1.2979,1.2979,0,0,1,17.541,15.64258Zm1.66309,1.10547H18.0957v2.17187h1.1084Z" style="fill: currentColor"/>
<path d="M5.68653,22.30351a2.00588,2.00588,0,0,1-2-2v-16A1.92585,1.92585,0,0,1,4.274,2.891a1.92585,1.92585,0,0,1,1.4125-.5875h8l6,6v5.66931h-2V9.30351h-5v-5h-7v16h9.74021v2Z" style="fill: currentColor"/>
<rect x="7.69809" y="10.43189" width="4.33778" height="0.79501" style="fill: currentColor"/>
<rect x="7.69809" y="12.16889" width="7.31192" height="1.21288" style="fill: currentColor"/>
<rect x="7.69809" y="17.14555" width="7.31192" height="1.21288" style="fill: currentColor"/>
<rect x="7.69809" y="14.32375" width="7.31192" height="0.57517" style="fill: currentColor"/>
<rect x="7.69809" y="15.79848" width="5.25578" height="0.4475" style="fill: currentColor"/>
</g>
</g>
</g>
</symbol>
</svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

@@ -1 +1,13 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="800" height="800" version="1.1" viewBox="0 0 512 512"><title>rename</title><symbol id="icon-rename" viewBox="0 0 512 512"><g id="Page-1" fill="none" fill-rule="evenodd" stroke="none" stroke-width="1"><g id="Combined-Shape" fill="currentColor"><path d="M362.666667,1.42108547e-14 L362.666667,21.3333333 L320,21.333 L320,362.666 L362.666667,362.666667 L362.666667,384 L320,383.999 L320,384 L298.666667,384 L298.666,383.999 L256,384 L256,362.666667 L298.666,362.666 L298.666,21.333 L256,21.3333333 L256,1.42108547e-14 L362.666667,1.42108547e-14 Z M426.666667,64 L426.666667,320 L341.333333,320 L341.333333,277.333333 L384,277.333333 L384,106.666667 L341.333333,106.666667 L341.333333,64 L426.666667,64 Z M277.333333,64 L277.333333,320 L3.55271368e-14,320 L3.55271368e-14,64 L277.333333,64 Z M179.2,89.6 L149.333333,89.6 L149.333333,234.666667 C149.333333,248 148.5,256.333333 147.875,264.354167 L147.792993,265.422171 L147.792993,265.422171 L147.714003,266.48894 C147.417695,270.579012 147.2,274.696296 147.2,279.466667 L147.2,279.466667 L177.066667,279.466667 L177.066667,260.266667 C184.941497,273.926888 199.708077,282.130544 215.466667,281.6 C229.540046,281.805757 242.921593,275.508559 251.733333,264.533333 C263.162478,248.989677 269.832496,230.461848 270.933333,211.2 C270.933333,170.666667 249.6,142.933333 217.6,142.933333 C202.507405,142.999748 188.308689,150.099106 179.2,162.133333 L179.2,162.133333 L179.2,89.6 Z M119.466667,162.133333 C107.961824,149.843793 91.4322333,143.546807 74.6666667,145.066667 C57.6785115,144.485924 40.8138255,148.15216 25.6,155.733333 L25.6,155.733333 L34.1333333,177.066667 C45.3979052,171.147831 57.7246848,167.522308 70.4,166.4 C78.5613135,165.511423 86.6853595,168.371259 92.4903835,174.176283 C98.2954074,179.981307 101.155244,188.105353 100.266667,196.266667 L100.266667,196.266667 L100.266667,198.4 L78.9333333,198.4 C65.8181975,197.679203 52.705771,199.864608 40.5333333,204.8 C26.2806563,210.950309 17.6507691,225.621117 19.2,241.066667 C19.0625857,252.057651 23.6679763,262.574827 31.8381493,269.927982 C40.0083223,277.281138 50.9508304,280.757072 61.8666667,279.466667 C77.2795695,280.291768 92.2192911,274.001359 102.4,262.4 L102.4,262.4 L102.4,277.333333 L130.133333,277.333333 C128.292479,266.054406 127.577851,254.620365 128,243.2 L128,243.2 L128,204.8 C129.999138,190.023932 126.995128,175.003882 119.466667,162.133333 Z M98.1333333,213.333333 L98.1333333,238.933333 C92.082572,249.988391 80.836024,257.218314 68.2666667,258.133333 C63.0655139,258.520242 57.9538681,256.621996 54.2659359,252.934064 C50.5780036,249.246132 48.6797582,244.134486 49.0666667,238.933333 C49.0666667,224 59.7333333,215.466667 85.3333333,213.333333 L85.3333333,213.333333 L98.1333333,213.333333 Z M209.066667,166.4 C226.133333,166.4 238.933333,183.466667 238.933333,211.2 C238.933333,238.933333 228.266667,256 211.2,256 C197.298049,255.69869 184.825037,247.383349 179.2,234.666667 L179.2,234.666667 L179.2,187.733333 C185.154203,176.240507 196.263981,168.304951 209.066667,166.4 Z" transform="translate(42.666667, 64.000000)"/></g></g></symbol></svg>
<?xml version="1.0" encoding="UTF-8"?>
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
<svg width="800px" height="800px" viewBox="0 0 512 512" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<title>rename</title>
<symbol id="icon-rename" viewBox="0 0 512 512">
<g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
<g id="Combined-Shape" fill="currentColor" transform="translate(42.666667, 64.000000)">
<path d="M362.666667,1.42108547e-14 L362.666667,21.3333333 L320,21.333 L320,362.666 L362.666667,362.666667 L362.666667,384 L320,383.999 L320,384 L298.666667,384 L298.666,383.999 L256,384 L256,362.666667 L298.666,362.666 L298.666,21.333 L256,21.3333333 L256,1.42108547e-14 L362.666667,1.42108547e-14 Z M426.666667,64 L426.666667,320 L341.333333,320 L341.333333,277.333333 L384,277.333333 L384,106.666667 L341.333333,106.666667 L341.333333,64 L426.666667,64 Z M277.333333,64 L277.333333,320 L3.55271368e-14,320 L3.55271368e-14,64 L277.333333,64 Z M179.2,89.6 L149.333333,89.6 L149.333333,234.666667 C149.333333,248 148.5,256.333333 147.875,264.354167 L147.792993,265.422171 L147.792993,265.422171 L147.714003,266.48894 C147.417695,270.579012 147.2,274.696296 147.2,279.466667 L147.2,279.466667 L177.066667,279.466667 L177.066667,260.266667 C184.941497,273.926888 199.708077,282.130544 215.466667,281.6 C229.540046,281.805757 242.921593,275.508559 251.733333,264.533333 C263.162478,248.989677 269.832496,230.461848 270.933333,211.2 C270.933333,170.666667 249.6,142.933333 217.6,142.933333 C202.507405,142.999748 188.308689,150.099106 179.2,162.133333 L179.2,162.133333 L179.2,89.6 Z M119.466667,162.133333 C107.961824,149.843793 91.4322333,143.546807 74.6666667,145.066667 C57.6785115,144.485924 40.8138255,148.15216 25.6,155.733333 L25.6,155.733333 L34.1333333,177.066667 C45.3979052,171.147831 57.7246848,167.522308 70.4,166.4 C78.5613135,165.511423 86.6853595,168.371259 92.4903835,174.176283 C98.2954074,179.981307 101.155244,188.105353 100.266667,196.266667 L100.266667,196.266667 L100.266667,198.4 L78.9333333,198.4 C65.8181975,197.679203 52.705771,199.864608 40.5333333,204.8 C26.2806563,210.950309 17.6507691,225.621117 19.2,241.066667 C19.0625857,252.057651 23.6679763,262.574827 31.8381493,269.927982 C40.0083223,277.281138 50.9508304,280.757072 61.8666667,279.466667 C77.2795695,280.291768 92.2192911,274.001359 102.4,262.4 L102.4,262.4 L102.4,277.333333 L130.133333,277.333333 C128.292479,266.054406 127.577851,254.620365 128,243.2 L128,243.2 L128,204.8 C129.999138,190.023932 126.995128,175.003882 119.466667,162.133333 Z M98.1333333,213.333333 L98.1333333,238.933333 C92.082572,249.988391 80.836024,257.218314 68.2666667,258.133333 C63.0655139,258.520242 57.9538681,256.621996 54.2659359,252.934064 C50.5780036,249.246132 48.6797582,244.134486 49.0666667,238.933333 C49.0666667,224 59.7333333,215.466667 85.3333333,213.333333 L85.3333333,213.333333 L98.1333333,213.333333 Z M209.066667,166.4 C226.133333,166.4 238.933333,183.466667 238.933333,211.2 C238.933333,238.933333 228.266667,256 211.2,256 C197.298049,255.69869 184.825037,247.383349 179.2,234.666667 L179.2,234.666667 L179.2,187.733333 C185.154203,176.240507 196.263981,168.304951 209.066667,166.4 Z">
</path>
</g>
</g>
</symbol>
</svg>

Before

Width:  |  Height:  |  Size: 3.1 KiB

After

Width:  |  Height:  |  Size: 3.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.3 KiB

After

Width:  |  Height:  |  Size: 6.1 KiB

@@ -4,8 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.lang.reflect.Field;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
@@ -24,10 +22,7 @@ class MultipartConfigurationTest {
void setUp() throws Exception {
// Manually constructed config with a mocked service, so Spring env overrides do not apply.
uploadLimitService = mock(UploadLimitService.class);
configuration = new MultipartConfiguration();
Field field = MultipartConfiguration.class.getDeclaredField("uploadLimitService");
field.setAccessible(true);
field.set(configuration, uploadLimitService);
configuration = new MultipartConfiguration(uploadLimitService);
}
@Nested
@@ -84,12 +84,6 @@ class MergeControllerGapTest {
return (MultipartFile[]) m.invoke(null, files, fileOrder);
}
private String[] parseClientFileIds(String value) throws Exception {
Method m = MergeController.class.getDeclaredMethod("parseClientFileIds", String.class);
m.setAccessible(true);
return (String[]) m.invoke(mergeController, value);
}
private long getPdfDateTimeSafe(MultipartFile file) throws Exception {
Method m =
MergeController.class.getDeclaredMethod("getPdfDateTimeSafe", MultipartFile.class);
@@ -331,58 +325,6 @@ class MergeControllerGapTest {
}
}
// ---- parseClientFileIds -------------------------------------------------
@Nested
@DisplayName("parseClientFileIds")
class ParseClientFileIds {
@Test
@DisplayName("null input returns empty array")
void nullReturnsEmpty() throws Exception {
assertEquals(0, parseClientFileIds(null).length);
}
@Test
@DisplayName("blank input returns empty array")
void blankReturnsEmpty() throws Exception {
assertEquals(0, parseClientFileIds(" ").length);
}
@Test
@DisplayName("empty JSON array returns empty array")
void emptyArrayReturnsEmpty() throws Exception {
assertEquals(0, parseClientFileIds("[]").length);
assertEquals(0, parseClientFileIds("[ ]").length);
}
@Test
@DisplayName("non-array text returns empty array")
void nonArrayReturnsEmpty() throws Exception {
assertEquals(0, parseClientFileIds("not-an-array").length);
}
@Test
@DisplayName("parses quoted, comma-separated ids and strips surrounding quotes")
void parsesQuotedIds() throws Exception {
String[] result = parseClientFileIds("[\"id1\", \"id2\",\"id3\"]");
assertArrayEquals(new String[] {"id1", "id2", "id3"}, result);
}
@Test
@DisplayName("parses unquoted ids as-is after trimming")
void parsesUnquotedIds() throws Exception {
String[] result = parseClientFileIds("[a, b , c]");
assertArrayEquals(new String[] {"a", "b", "c"}, result);
}
@Test
@DisplayName("single element array yields a one-element result")
void singleElement() throws Exception {
assertArrayEquals(new String[] {"only"}, parseClientFileIds("[\"only\"]"));
}
}
// ---- reorderFilesByProvidedOrder ----------------------------------------
@Nested
@@ -0,0 +1,66 @@
package stirling.software.SPDF.controller.api;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.http.ResponseEntity;
import stirling.software.SPDF.service.SharedSignatureService;
import stirling.software.common.configuration.RuntimePathConfig;
import stirling.software.common.model.ApplicationProperties;
import tools.jackson.databind.ObjectMapper;
class UIDataControllerTest {
@TempDir Path tempDir;
@Test
void getPipelineData_usesEachSourceFilenameWhenJsonContentIsIdentical() throws Exception {
Path configDir = tempDir.resolve("defaultWebUIConfigs");
Files.createDirectories(configDir);
String configJson = "{\"steps\":[]}";
Files.writeString(configDir.resolve("first-config.json"), configJson);
Files.writeString(configDir.resolve("second-config.json"), configJson);
ApplicationProperties applicationProperties = mock(ApplicationProperties.class);
SharedSignatureService signatureService = mock(SharedSignatureService.class);
RuntimePathConfig runtimePathConfig = mock(RuntimePathConfig.class);
when(runtimePathConfig.getPipelineDefaultWebUiConfigs()).thenReturn(configDir.toString());
UIDataController controller =
new UIDataController(
applicationProperties,
signatureService,
null,
new DefaultResourceLoader(),
runtimePathConfig,
new ObjectMapper());
ResponseEntity<UIDataController.PipelineData> response = controller.getPipelineData();
assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
UIDataController.PipelineData body = response.getBody();
assertThat(body).isNotNull();
List<Map<String, String>> configsWithNames = body.getPipelineConfigsWithNames();
assertThat(configsWithNames).hasSize(2);
assertThat(configsWithNames)
.extracting(entry -> entry.get("name"))
.containsExactlyInAnyOrder("first-config", "second-config");
assertThat(configsWithNames)
.extracting(entry -> entry.get("json"))
.containsOnly(configJson);
}
}
@@ -204,7 +204,7 @@ class ConvertOfficeControllerTest {
inv -> {
// unoconvert writes directly to the output path (last arg)
List<String> command = inv.getArgument(0);
Path out = Path.of(command.get(command.size() - 1));
Path out = Path.of(command.getLast());
Files.writeString(out, "%PDF-1.4 produced");
return result;
});
@@ -239,7 +239,7 @@ class ConvertOfficeControllerTest {
inv -> {
// soffice writes <basename>.pdf into the --outdir (workDir)
List<String> command = inv.getArgument(0);
Path inputPath = Path.of(command.get(command.size() - 1));
Path inputPath = Path.of(command.getLast());
Path out = inputPath.getParent().resolve("report.pdf");
Files.writeString(out, "%PDF soffice");
return result;
@@ -311,7 +311,7 @@ class ConvertOfficeControllerTest {
.thenAnswer(
inv -> {
List<String> command = inv.getArgument(0);
Path inputPath = Path.of(command.get(command.size() - 1));
Path inputPath = Path.of(command.getLast());
Path out = inputPath.getParent().resolve("report.pdf");
Files.write(out, new byte[0]);
return result;
@@ -344,7 +344,7 @@ class ConvertOfficeControllerTest {
.thenAnswer(
inv -> {
List<String> command = inv.getArgument(0);
Path inputPath = Path.of(command.get(command.size() - 1));
Path inputPath = Path.of(command.getLast());
Path out = inputPath.getParent().resolve("page.pdf");
Files.writeString(out, "%PDF html");
return result;
@@ -398,7 +398,7 @@ class ConvertOfficeControllerTest {
.thenAnswer(
inv -> {
List<String> command = inv.getArgument(0);
Path inputPath = Path.of(command.get(command.size() - 1));
Path inputPath = Path.of(command.getLast());
Path out = inputPath.getParent().resolve("report.pdf");
Files.writeString(out, "%PDF produced");
return result;
@@ -197,7 +197,7 @@ class ConvertPDFToPDFAMoreTest {
// qpdf normalize/clean writes its (last-arg) output file
if (command.contains("--normalize-content=y")) {
// qpdf produced file is the last argument
Path out = Path.of(command.get(command.size() - 1));
Path out = Path.of(command.getLast());
Files.write(out, simplePdfBytes());
}
return okResult;
@@ -262,7 +262,7 @@ class ConvertPdfToVideoControllerTest {
assertTrue(command.contains("+faststart"));
assertFalse(command.contains("libvpx-vp9"));
// Output path is always the last argument.
assertEquals(backing.getAbsolutePath(), command.get(command.size() - 1));
assertEquals(backing.getAbsolutePath(), command.getLast());
}
@Test
@@ -278,7 +278,7 @@ class ConvertPdfToVideoControllerTest {
assertTrue(command.contains("30"));
assertFalse(command.contains("libx264"));
assertFalse(command.contains("+faststart"));
assertEquals(backing.getAbsolutePath(), command.get(command.size() - 1));
assertEquals(backing.getAbsolutePath(), command.getLast());
}
@Test
@@ -195,7 +195,7 @@ class CompressControllerMoreTest {
// The qpdf output path is the last argument of the command.
private static Path qpdfOutputPath(List<String> command) {
return Path.of(command.get(command.size() - 1));
return Path.of(command.getLast());
}
/** Stub gs to write a valid PDF to its output file and report success. */
@@ -162,7 +162,7 @@ class RemoveImagesControllerTest {
/** Counts every PDImageXObject reachable through page + nested form resources. */
private int countImagesInSavedOutput() throws IOException {
assertFalse(savedTempFiles.isEmpty(), "expected the controller to create a temp file");
File out = savedTempFiles.get(savedTempFiles.size() - 1);
File out = savedTempFiles.getLast();
try (PDDocument doc = Loader.loadPDF(out)) {
int count = 0;
for (PDPage page : doc.getPages()) {
@@ -245,7 +245,7 @@ class RemoveImagesControllerTest {
assertEquals(0, countImagesInSavedOutput());
// page count must be preserved
File out = savedTempFiles.get(savedTempFiles.size() - 1);
File out = savedTempFiles.getLast();
try (PDDocument result = Loader.loadPDF(out)) {
assertEquals(3, result.getNumberOfPages());
}
@@ -98,12 +98,8 @@ class RepairControllerMoreTest {
}
}
/**
* Writes a valid PDF to the path at the given command index, mimicking a successful tool run.
*/
private static void writeValidPdfTo(List<String> command, int outputPathIndex)
throws Exception {
Path out = Path.of(command.get(outputPathIndex));
/** Writes a valid PDF to the given output path, mimicking a successful tool run. */
private static void writeValidPdfTo(Path out) throws Exception {
byte[] pdf = buildPdfBytes(1);
Files.write(out, pdf);
}
@@ -133,7 +129,7 @@ class RepairControllerMoreTest {
.thenAnswer(
inv -> {
List<String> cmd = inv.getArgument(0);
writeValidPdfTo(cmd, 2);
writeValidPdfTo(Path.of(cmd.get(2)));
return okResult;
});
@@ -176,7 +172,7 @@ class RepairControllerMoreTest {
.thenAnswer(
inv -> {
List<String> cmd = inv.getArgument(0);
writeValidPdfTo(cmd, cmd.size() - 1);
writeValidPdfTo(Path.of(cmd.getLast()));
return okResult;
});
@@ -216,7 +212,7 @@ class RepairControllerMoreTest {
.thenAnswer(
inv -> {
List<String> cmd = inv.getArgument(0);
writeValidPdfTo(cmd, cmd.size() - 1);
writeValidPdfTo(Path.of(cmd.getLast()));
return okResult;
});
@@ -256,7 +252,7 @@ class RepairControllerMoreTest {
.thenAnswer(
inv -> {
List<String> cmd = inv.getArgument(0);
writeValidPdfTo(cmd, cmd.size() - 1);
writeValidPdfTo(Path.of(cmd.getLast()));
return okResult;
});
@@ -299,18 +299,6 @@ class RedactControllerMoreTest {
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
}
@Test
@DisplayName("null listOfText throws an illegal-argument error before any load")
void nullPatternsThrows() throws Exception {
RedactPdfRequest request = new RedactPdfRequest();
request.setFileInput(pdfFile(new byte[] {1, 2, 3}));
request.setListOfText(null);
assertThatThrownBy(() -> controller.redactPdf(request))
.isInstanceOf(RuntimeException.class);
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
}
@Test
@DisplayName("null file input is reported as a failure")
void nullFileThrows() {
@@ -49,15 +49,7 @@ class UploadLimitServiceTest {
systemProps = mock(ApplicationProperties.System.class);
when(applicationProperties.getSystem()).thenReturn(systemProps);
uploadLimitService = new UploadLimitService();
// inject mock
try {
var field = UploadLimitService.class.getDeclaredField("applicationProperties");
field.setAccessible(true);
field.set(uploadLimitService, applicationProperties);
} catch (ReflectiveOperationException e) {
throw new RuntimeException(e);
}
uploadLimitService = new UploadLimitService(applicationProperties);
}
@ParameterizedTest(name = "getReadableUploadLimit case #{index}: rawValue={0}, expected={1}")
@@ -142,9 +142,6 @@ public class AuditAspect {
auditService.addTimingData(
auditData, startTime, resp, auditedAnnotation.level(), isHttpRequest);
// Merge controller-set policy context + the internal-automation marker onto the event.
auditService.addAutomationContext(auditData, req);
// Resolve the event type based on annotation and context
String httpMethod = null;
String path = null;
@@ -1,18 +0,0 @@
package stirling.software.proprietary.audit;
/**
* Request-scoped keys a controller can set to enrich its own audit event with context the generic
* aspect can't infer from the HTTP request alone (e.g. the policy a pipeline run belongs to). The
* aspect copies these into the audit data in its {@code finally} block, after the controller body
* has run. See {@code AuditService#addAutomationContext}.
*/
public final class AuditContext {
/** Request attribute: the name of the policy/pipeline a run executes. */
public static final String REQ_ATTR_POLICY_NAME = "stirling.audit.policyName";
/** Request attribute: the ordered tool endpoint paths a run executes. */
public static final String REQ_ATTR_POLICY_STEPS = "stirling.audit.policySteps";
private AuditContext() {}
}
@@ -205,10 +205,6 @@ public class ControllerAuditAspect {
// Call auditService but with isHttpRequest=true to skip additional timing
auditService.addTimingData(data, start, resp, level, true);
// Merge controller-set policy context + the internal-automation marker (set after
// the body ran, so it must happen here rather than with the pre-proceed HTTP data).
auditService.addAutomationContext(data, req);
// Resolve the event type using the unified method
AuditEventType eventType =
auditService.resolveEventType(
@@ -133,45 +133,29 @@ public final class S3Clients {
* storage.s3.allow-private-endpoints=true}.
*/
static void validateEndpointHost(URI endpoint, boolean allowPrivate) {
validateEndpointHost(
endpoint,
allowPrivate,
"storage.s3.endpoint",
"set storage.s3.allow-private-endpoints=true to opt in"
+ " (e.g. for MinIO or in-cluster S3).");
}
/**
* The same private-address guard for S3 endpoints configured outside the {@code storage.s3.*}
* block (e.g. per-source policy config), with the setting named in messages supplied by the
* caller.
*/
public static void validateEndpointHost(
URI endpoint, boolean allowPrivate, String settingName, String optInHint) {
if (allowPrivate) {
return;
}
String host = endpoint.getHost();
if (host == null || host.isBlank()) {
throw new IllegalStateException(settingName + " must include a host: " + endpoint);
throw new IllegalStateException("storage.s3.endpoint must include a host: " + endpoint);
}
InetAddress[] addresses;
try {
addresses = InetAddress.getAllByName(host);
} catch (UnknownHostException e) {
throw new IllegalStateException(
"Unable to resolve " + settingName + " host '" + host + "'", e);
"Unable to resolve storage.s3.endpoint host '" + host + "'", e);
}
for (InetAddress address : addresses) {
if (isPrivateOrLocal(address)) {
throw new IllegalStateException(
settingName
+ " host '"
"storage.s3.endpoint host '"
+ host
+ "' resolves to private/link-local address "
+ address.getHostAddress()
+ "; "
+ optInHint);
+ "; set storage.s3.allow-private-endpoints=true to opt in"
+ " (e.g. for MinIO or in-cluster S3).");
}
}
}
@@ -143,19 +143,12 @@ public class AuditRestController {
@RequestParam(value = "period", defaultValue = "week") String period) {
// Calculate days based on period
int days;
switch (period.toLowerCase()) {
case "day":
days = 1;
break;
case "month":
days = 30;
break;
case "week":
default:
days = 7;
break;
}
int days =
switch (period.toLowerCase()) {
case "day" -> 1;
case "month" -> 30;
default -> 7;
};
// Get events from the specified period
Instant startDate = Instant.now().minus(java.time.Duration.ofDays(days));
@@ -269,19 +262,12 @@ public class AuditRestController {
@RequestParam(value = "period", defaultValue = "week") String period) {
// Calculate days based on period
int days;
switch (period.toLowerCase()) {
case "day":
days = 1;
break;
case "month":
days = 30;
break;
case "week":
default:
days = 7;
break;
}
int days =
switch (period.toLowerCase()) {
case "day" -> 1;
case "month" -> 30;
default -> 7;
};
// Get events from the specified period and previous period
Instant now = Instant.now();
@@ -754,7 +740,7 @@ public class AuditRestController {
List<Map<String, Object>> files =
(List<Map<String, Object>>) eventData.get("files");
if (files != null && !files.isEmpty()) {
Map<String, Object> firstFile = files.get(0);
Map<String, Object> firstFile = files.getFirst();
data.put("documentname", String.valueOf(firstFile.getOrDefault("name", "")));
data.put("author", String.valueOf(firstFile.getOrDefault("pdfAuthor", "")));
data.put("filehash", String.valueOf(firstFile.getOrDefault("fileHash", "")));
@@ -4,7 +4,6 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import org.springframework.context.annotation.Profile;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -39,9 +38,6 @@ import stirling.software.proprietary.security.database.repository.UserRepository
@PreAuthorize("hasRole('ADMIN')")
@RequiredArgsConstructor
@EnterpriseEndpoint
// Self-hosted only: counts are server-wide. On SaaS this endpoint is owned by the team-scoped
// SaasFleetUsageController (@Profile("saas")) so one backend can't leak another tenant's usage.
@Profile("!saas")
public class FleetUsageController {
private final PersistentAuditEventRepository auditRepository;
@@ -1,31 +0,0 @@
package stirling.software.proprietary.integration.crypto;
import jakarta.persistence.AttributeConverter;
import jakarta.persistence.Converter;
/**
* {@link EncryptedStringConverter} for columns that held plaintext before encryption shipped:
* writes are always encrypted, but a stored value that is not valid ciphertext is returned as-is,
* so pre-encryption rows keep loading and become encrypted on their next save. The discrimination
* is exact for JSON payloads, which can never be mistaken for ciphertext ('{' is not in the Base64
* alphabet). The trade-off is that a genuinely corrupted ciphertext surfaces as garbage to the
* caller's parser instead of failing here.
*/
@Converter
public class LenientEncryptedStringConverter implements AttributeConverter<String, String> {
@Override
public String convertToDatabaseColumn(String attribute) {
return CredentialEncryption.encrypt(attribute);
}
@Override
public String convertToEntityAttribute(String dbData) {
try {
return CredentialEncryption.decrypt(dbData);
} catch (IllegalArgumentException | IllegalStateException e) {
// Not ciphertext: legacy plaintext from before encryption shipped.
return dbData;
}
}
}
@@ -43,10 +43,6 @@ public class IntegrationConfigService {
private final OwnershipService ownership;
private final SecretMasker secretMasker;
private final ResourceGrantRepository grantRepository;
// Bean-discovered extension points: features that understand a type contribute its config
// schema and report what still references a config, without this module depending on them.
private final List<IntegrationConfigValidator> validators;
private final List<IntegrationConfigUsageCheck> usageChecks;
// ---- commands ----
@@ -70,21 +66,13 @@ public class IntegrationConfigService {
? DefaultAccessPolicy.EXPLICIT_ONLY
: request.defaultAccess());
// TEAM scope may omit the team id: default to the caller's own team so clients (the
// portal) need not know it. assignOwnership still enforces admin-or-leader of that team.
Long ownerTeamId = request.ownerTeamId();
if (ownerTeamId == null && scope == OwnerScope.TEAM && currentUser.getTeam() != null) {
ownerTeamId = currentUser.getTeam().getId();
}
ownership.assignOwnership(
cfg,
scope,
ownerTeamId,
request.ownerTeamId(),
currentUser,
() -> lockedServerExists(cfg.getIntegrationType()));
Map<String, Object> config = secretMasker.sanitize(request.config());
validateConfig(cfg.getIntegrationType(), config);
cfg.setConfig(writeJson(config));
cfg.setConfig(writeJson(secretMasker.sanitize(request.config())));
return repository.save(cfg);
}
@@ -113,10 +101,8 @@ public class IntegrationConfigService {
cfg.setDefaultAccess(request.defaultAccess());
}
if (request.config() != null) {
Map<String, Object> merged =
secretMasker.merge(readJson(cfg.getConfig()), request.config());
validateConfig(cfg.getIntegrationType(), merged);
cfg.setConfig(writeJson(merged));
cfg.setConfig(
writeJson(secretMasker.merge(readJson(cfg.getConfig()), request.config())));
}
return repository.save(cfg);
}
@@ -127,15 +113,6 @@ public class IntegrationConfigService {
if (!ownership.canManage(TYPE, cfg, currentUser)) {
throw forbidden("You cannot manage this integration");
}
// Refuse to pull a connection out from under whatever still references it.
List<String> usages =
usageChecks.stream()
.flatMap(check -> check.usagesOf(cfg.getId()).stream())
.toList();
if (!usages.isEmpty()) {
throw new ResponseStatusException(
HttpStatus.CONFLICT, "Integration is in use by: " + String.join(", ", usages));
}
// Drop grants sharing this config so they do not dangle as dead rows.
grantRepository.deleteByResourceTypeAndResourceId(TYPE, String.valueOf(cfg.getId()));
repository.delete(cfg);
@@ -211,19 +188,6 @@ public class IntegrationConfigService {
// ---- integration-specific glue ----
/** Runs every registered validator for the type; unknown types save free-form. */
private void validateConfig(IntegrationType type, Map<String, Object> config) {
for (IntegrationConfigValidator validator : validators) {
if (validator.type() == type) {
try {
validator.validate(config == null ? Map.of() : config);
} catch (IllegalArgumentException e) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
}
}
}
}
/** A non-admin can't create a personal config of a type an admin has locked at server scope. */
private boolean lockedServerExists(IntegrationType type) {
return repository.findByScope(OwnerScope.SERVER).stream()
@@ -1,15 +0,0 @@
package stirling.software.proprietary.integration.service;
import java.util.List;
/**
* Reports what still references an integration config, so deletion can be refused instead of
* pulling a connection out from under a live consumer. Implementations are beans discovered by
* {@link IntegrationConfigService} (e.g. the policy subsystem reporting sources and pipelines that
* reference a connection).
*/
public interface IntegrationConfigUsageCheck {
/** Human-readable labels of everything still using the config; empty when unreferenced. */
List<String> usagesOf(long configId);
}
@@ -1,23 +0,0 @@
package stirling.software.proprietary.integration.service;
import java.util.Map;
import stirling.software.proprietary.integration.model.IntegrationType;
/**
* Validates one integration type's config map at save time. Implementations are beans discovered by
* {@link IntegrationConfigService}, so the feature that understands a type (e.g. the policy S3
* backend) owns its schema without the integration module depending on it. Types with no registered
* validator save free-form.
*/
public interface IntegrationConfigValidator {
/** The type this validator understands. */
IntegrationType type();
/**
* Validates the config as it will be stored (secrets already sanitized/merged, so values are
* real, never the redaction mask). Throws {@link IllegalArgumentException} on bad config.
*/
void validate(Map<String, Object> config);
}
@@ -17,9 +17,6 @@ public class InfraAuditSummary {
@Schema(description = "Total events in the returned window", example = "40")
private int totalEvents;
@Schema(description = "Policy-run events", example = "3")
private int policy;
@Schema(description = "Processing-category events", example = "24")
private int processing;
@@ -23,8 +23,6 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
@@ -34,7 +32,6 @@ import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
@@ -45,18 +42,14 @@ import stirling.software.common.model.job.JobResponse;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.proprietary.audit.AuditContext;
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.PolicyValidator;
import stirling.software.proprietary.policy.engine.SweepOutcome;
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
@@ -65,15 +58,12 @@ import stirling.software.proprietary.policy.model.PolicyRunView;
import stirling.software.proprietary.policy.overview.PoliciesOverviewResponse;
import stirling.software.proprietary.policy.overview.PolicyOverviewService;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
import stirling.software.proprietary.policy.source.EditorSource;
import stirling.software.proprietary.policy.source.SourceAccessGuard;
import stirling.software.proprietary.policy.source.SourceDocCounter;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
import stirling.software.proprietary.policy.trigger.TriggerInfo;
import stirling.software.proprietary.util.SecretMasker;
/**
* Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
@@ -93,7 +83,6 @@ public class PolicyController {
private final PolicyStore policyStore;
private final SourceStore sourceStore;
private final SourceAccessGuard sourceAccessGuard;
private final SourceDocCounter docCounter;
private final PolicyValidator policyValidator;
private final PolicyAccessGuard policyAccessGuard;
private final PolicyManagementAuthority policyManagementAuthority;
@@ -118,14 +107,11 @@ public class PolicyController {
@RequestPart("json") PipelineDefinition definition,
@Valid @ModelAttribute PolicyRunFiles files)
throws IOException {
stampPolicyAudit(definition);
requireRunnable(definition);
validateAdHocOutput(definition);
PolicyInputs inputs = toInputs(files);
PolicyRunHandle handle =
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP);
recordEditorDocs(inputs);
return ResponseEntity.accepted().body(new JobResponse<>(true, handle.runId(), null));
String runId =
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
}
@PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@@ -139,9 +125,7 @@ public class PolicyController {
@RequestPart("json") PipelineDefinition definition,
@Valid @ModelAttribute PolicyRunFiles files)
throws IOException {
stampPolicyAudit(definition);
requireRunnable(definition);
validateAdHocOutput(definition);
PolicyInputs inputs = toInputs(files);
SseEmitter emitter =
@@ -149,7 +133,6 @@ public class PolicyController {
emitter.onError(e -> log.warn("Policy run SSE emitter error", e));
PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter));
recordEditorDocs(inputs);
// whenComplete runs on the worker thread after the run finishes, so the terminal event
// never races the step events.
handle.completion()
@@ -219,7 +202,7 @@ public class PolicyController {
+ " assigned; returns the stored policy with its id.")
public ResponseEntity<Policy> savePolicy(@RequestBody Policy policy) {
requirePolicyEditingAllowed();
Policy owned = withStoredOutputSecrets(resolveOwnership(policy));
Policy owned = resolveOwnership(policy);
requireAccessibleSources(owned);
try {
policyValidator.validate(owned);
@@ -230,7 +213,7 @@ public class PolicyController {
// Re-sync trigger registrations now so a new/changed folder-watch policy starts being
// watched immediately instead of after the next reconcile sweep.
policyTriggerManager.notifyPoliciesChanged();
return ResponseEntity.ok(withMaskedOutputSecrets(saved));
return ResponseEntity.ok(saved);
}
@PutMapping("/order")
@@ -299,50 +282,6 @@ public class PolicyController {
teamId);
}
/** Output secrets never leave the server: reads return the redaction sentinel instead. */
private static Policy withMaskedOutputSecrets(Policy policy) {
return withOutput(
policy,
new OutputSpec(
policy.output().type(), SecretMasker.mask(policy.output().options())));
}
/**
* An edit that round-trips a masked read sends output secrets back as the sentinel; restore
* them from the stored policy so saving without re-typing keeps them (validation then runs
* against the real values).
*/
private Policy withStoredOutputSecrets(Policy incoming) {
if (incoming.id() == null || incoming.id().isBlank()) {
return incoming;
}
return policyStore
.get(incoming.id())
.map(
existing ->
withOutput(
incoming,
new OutputSpec(
incoming.output().type(),
SecretMasker.restoreRedacted(
incoming.output().options(),
existing.output().options()))))
.orElse(incoming);
}
private static Policy withOutput(Policy policy, OutputSpec output) {
return new Policy(
policy.id(),
policy.name(),
policy.owner(),
policy.enabled(),
policy.trigger(),
policy.sourceIds(),
policy.steps(),
output,
policy.teamId());
}
/**
* Creating, editing, pausing/resuming, and deleting policies requires the editor role for the
* caller's team — a team leader on SaaS (see {@link PolicyManagementAuthority}); the global
@@ -367,14 +306,9 @@ public class PolicyController {
@GetMapping
@Operation(
summary = "List policies",
description =
"Lists the policies belonging to the caller's team. Secret-bearing output"
+ " options are returned as a redaction sentinel, never their stored"
+ " values.")
description = "Lists the policies belonging to the caller's team.")
public List<Policy> listPolicies() {
return policyAccessGuard.visibleFrom(policyStore).stream()
.map(PolicyController::withMaskedOutputSecrets)
.toList();
return policyAccessGuard.visibleFrom(policyStore);
}
@GetMapping("/overview")
@@ -403,17 +337,11 @@ public class PolicyController {
}
@GetMapping("/{policyId}")
@Operation(
summary = "Get a policy by id",
description =
"Secret-bearing output options are returned as a redaction sentinel, never"
+ " their stored values; an edit that sends the sentinel back keeps"
+ " them.")
@Operation(summary = "Get a policy by id")
public ResponseEntity<Policy> getPolicy(@PathVariable String policyId) {
return policyStore
.get(policyId)
.filter(policyAccessGuard::canAccess)
.map(PolicyController::withMaskedOutputSecrets)
.map(ResponseEntity::ok)
.orElseGet(() -> ResponseEntity.notFound().build());
}
@@ -473,7 +401,6 @@ public class PolicyController {
() ->
new ResponseStatusException(
HttpStatus.NOT_FOUND, "No policy: " + policyId));
stampPolicyAudit(policy.toDefinition());
PolicyInputs inputs = toInputs(files);
String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
@@ -485,10 +412,9 @@ public class PolicyController {
description =
"Pulls the policy's configured sources and runs the pipeline now, regardless of"
+ " the enabled flag (which only gates automatic triggering). Returns"
+ " the ids of the runs started (poll the run-status endpoint for each)"
+ " plus what the sweep skipped - already-processed, parked-by-failure,"
+ " and in-flight counts - so an empty result explains itself.")
public ResponseEntity<SweepOutcome> trigger(@PathVariable String policyId) {
+ " the ids of the runs started; poll the run-status endpoint for each."
+ " Empty when the sources yielded no work to do.")
public ResponseEntity<List<String>> trigger(@PathVariable String policyId) {
Policy policy =
policyStore
.get(policyId)
@@ -507,60 +433,6 @@ public class PolicyController {
}
}
/**
* Stamp this run's policy name and step endpoints onto the current request so the audit aspect
* can label the event as the policy it ran (rather than the generic {@code /run} endpoint) and
* record which tools it executed. No-op outside a web request.
*/
private static void stampPolicyAudit(PipelineDefinition definition) {
if (definition == null
|| !(RequestContextHolder.getRequestAttributes()
instanceof ServletRequestAttributes attrs)) {
return;
}
HttpServletRequest request = attrs.getRequest();
if (definition.name() != null && !definition.name().isBlank()) {
request.setAttribute(AuditContext.REQ_ATTR_POLICY_NAME, definition.name());
}
List<String> steps =
definition.steps().stream()
.map(PipelineStep::operation)
.filter(op -> op != null && !op.isBlank())
.toList();
if (!steps.isEmpty()) {
request.setAttribute(AuditContext.REQ_ATTR_POLICY_STEPS, steps);
}
}
/**
* Authorization-check an ad-hoc run's output while the caller's principal is present (this
* request thread). The worker thread that later delivers carries no security context, so an S3
* output's connection-access check would be skipped there; without this gate a caller could
* reference another tenant's connection by id and write to it (confused deputy). Stored
* policies are covered by save-time {@link PolicyValidator#validate} instead.
*/
private void validateAdHocOutput(PipelineDefinition definition) {
if (definition.output() == null) {
return;
}
try {
policyValidator.validateOutput(definition.output());
} catch (IllegalArgumentException e) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
}
}
/**
* Ad-hoc runs (AI / one-off pipelines) are still editor activity, so their supplied documents
* feed the same virtual editor source as stored editor policies, counted against the caller's
* team. A run with no primary documents (generator pipeline) records nothing.
*/
private void recordEditorDocs(PolicyInputs inputs) {
docCounter.record(
EditorSource.counterKey(sourceAccessGuard.currentTeamId()),
inputs.primary().size());
}
/**
* Turn the typed run files into engine {@link PolicyInputs}: the primary documents plus the
* named supporting-file store, where each asset's {@code key} is the name a step references
@@ -21,9 +21,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
import stirling.software.common.service.AutomationRunContext;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.InternalApiClient;
import stirling.software.common.service.InternalApiTimeoutException;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.service.JobQueue;
@@ -150,7 +148,6 @@ public class PolicyEngine {
runAsPrincipal(
billingPrincipal,
fileOwner,
definition.name(),
() -> runToCompletion(run, inputs, tracking, completion));
// One admission unit per run; steps run synchronously within it, so this gates heavy work
@@ -202,78 +199,63 @@ public class PolicyEngine {
PolicyProgressListener listener,
CompletableFuture<PolicyRun> completion) {
String runId = run.getRunId();
// One policy run = one automation run. Scope the run id on this worker thread (the async
// hop already happened) so every tool sub-step dispatched via InternalApiClient groups into
// a single charge, and two separate policy runs on the same document stay distinct charges.
try (AutomationRunContext.Scope runScope = AutomationRunContext.open(runId)) {
try {
run.markRunning();
PolicyExecutionResult result =
stepExecutor.execute(run.getDefinition(), inputs, listener);
OutputSpec output = run.getDefinition().output();
List<ResultFile> outputs =
sinkFor(output)
.deliver(
new OutputDelivery(runId, run.getPolicyId()),
result.files(),
output);
taskManager.setMultipleFileResults(runId, outputs);
taskManager.setComplete(runId);
run.complete(outputs);
} catch (PolicyInputRequiredException e) {
// Expected path: suspend rather than fail. Persist intermediates as fileIds so the
// run
// can resume after this worker thread is gone.
WaitState wait = suspend(e);
run.waitForInput(wait);
taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
} catch (InternalApiTimeoutException e) {
String message = toolTimeoutMessage(e);
log.error(
"Policy run {} timed out on {}: {}",
runId,
e.getEndpointPath(),
e.getMessage());
run.fail(message);
try {
run.markRunning();
PolicyExecutionResult result =
stepExecutor.execute(run.getDefinition(), inputs, listener);
OutputSpec output = run.getDefinition().output();
List<ResultFile> outputs =
sinkFor(output)
.deliver(
new OutputDelivery(runId, run.getPolicyId()),
result.files(),
output);
taskManager.setMultipleFileResults(runId, outputs);
taskManager.setComplete(runId);
run.complete(outputs);
} catch (PolicyInputRequiredException e) {
// Expected path: suspend rather than fail. Persist intermediates as fileIds so the run
// can resume after this worker thread is gone.
WaitState wait = suspend(e);
run.waitForInput(wait);
taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
} catch (InternalApiTimeoutException e) {
String message = toolTimeoutMessage(e);
log.error(
"Policy run {} timed out on {}: {}",
runId,
e.getEndpointPath(),
e.getMessage());
run.fail(message);
taskManager.setError(runId, message);
} catch (RestClientResponseException e) {
// A downstream tool call returned an error status. When it's a structured entitlement
// response (401/402 with a JSON `error` sentinel), surface that code onto the run so
// the
// client can react — e.g. pop the usage-limit modal — instead of only seeing a generic
// failure. We don't interpret the code here (that would couple this module to the saas
// billing layer); we just pass it through for the client to map. Other statuses fall
// through to the generic failure below.
String code = DownstreamEntitlementError.extractCode(e);
if (code != null) {
log.info("Policy run {} blocked by downstream entitlement gate ({})", runId, code);
String message = "Usage limit reached";
run.failWithCode(message, code, DownstreamEntitlementError.extractSubscribed(e));
taskManager.setError(runId, message);
} catch (RestClientResponseException e) {
// A downstream tool call returned an error status. When it's a structured
// entitlement
// response (401/402 with a JSON `error` sentinel), surface that code onto the run
// so
// the
// client can react — e.g. pop the usage-limit modal — instead of only seeing a
// generic
// failure. We don't interpret the code here (that would couple this module to the
// saas
// billing layer); we just pass it through for the client to map. Other statuses
// fall
// through to the generic failure below.
String code = DownstreamEntitlementError.extractCode(e);
if (code != null) {
log.info(
"Policy run {} blocked by downstream entitlement gate ({})",
runId,
code);
String message = "Usage limit reached";
run.failWithCode(
message, code, DownstreamEntitlementError.extractSubscribed(e));
taskManager.setError(runId, message);
} else {
String message = "Policy run failed: " + e.getMessage();
log.error("Policy run {} failed (downstream HTTP error)", runId, e);
run.fail(message);
taskManager.setError(runId, message);
}
} catch (Exception e) {
} else {
String message = "Policy run failed: " + e.getMessage();
log.error("Policy run {} failed", runId, e);
log.error("Policy run {} failed (downstream HTTP error)", runId, e);
run.fail(message);
taskManager.setError(runId, message);
} finally {
// Always resolve so stream/await callers unblock.
completion.complete(run);
}
} catch (Exception e) {
String message = "Policy run failed: " + e.getMessage();
log.error("Policy run {} failed", runId, e);
run.fail(message);
taskManager.setError(runId, message);
} finally {
// Always resolve so stream/await callers unblock.
completion.complete(run);
}
}
@@ -379,26 +361,18 @@ public class PolicyEngine {
* dispatch attributes (and charges) usage to that user. A null/blank principal runs as-is.
* Restores the previous MDC value afterward (defensive — worker threads aren't pooled).
*/
private static void runAsPrincipal(
String billingPrincipal, String fileOwner, String policyName, Runnable body) {
private static void runAsPrincipal(String billingPrincipal, String fileOwner, Runnable body) {
// Billing identity (MDC auditPrincipal) and output-file ownership (JobContext owner) are
// set
// independently: usage is charged to billingPrincipal, but stored output files are owned by
// fileOwner — the user who triggered an org-wide policy — so they can fetch their results.
// Either may be null (e.g. login disabled, or a trigger-fired run); each is applied only
// when present and restored afterward (defensive — worker threads aren't pooled). The
// policy
// name rides MDC too so each tool step's loopback dispatch (InternalApiClient) can forward
// it as a header, letting the audit tie the step back to its policy.
// when present and restored afterward (defensive — worker threads aren't pooled).
String previousPrincipal = MDC.get(AUDIT_PRINCIPAL_MDC_KEY);
String previousPolicyName = MDC.get(InternalApiClient.POLICY_NAME_MDC_KEY);
String previousOwner = JobContext.getOwner();
if (billingPrincipal != null && !billingPrincipal.isBlank()) {
MDC.put(AUDIT_PRINCIPAL_MDC_KEY, billingPrincipal);
}
if (policyName != null && !policyName.isBlank()) {
MDC.put(InternalApiClient.POLICY_NAME_MDC_KEY, policyName);
}
if (fileOwner != null && !fileOwner.isBlank()) {
JobContext.setOwner(fileOwner);
}
@@ -410,11 +384,6 @@ public class PolicyEngine {
} else {
MDC.remove(AUDIT_PRINCIPAL_MDC_KEY);
}
if (previousPolicyName != null) {
MDC.put(InternalApiClient.POLICY_NAME_MDC_KEY, previousPolicyName);
} else {
MDC.remove(InternalApiClient.POLICY_NAME_MDC_KEY);
}
JobContext.setOwner(previousOwner);
}
}
@@ -131,7 +131,7 @@ public class PolicyExecutor {
// One call over all inputs. The outputs derive from a single input only when exactly
// one entered; otherwise (a genuine merge) there is no single source.
ToolResult r = callEndpoint(step, inputFiles, supportingFiles);
Integer origin = inputOrigins.size() == 1 ? inputOrigins.get(0) : null;
Integer origin = inputOrigins.size() == 1 ? inputOrigins.getFirst() : null;
for (Resource file : r.files()) {
files.add(file);
origins.add(origin);
@@ -21,7 +21,6 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
import stirling.software.proprietary.policy.source.EditorSource;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceDocCounter;
import stirling.software.proprietary.policy.source.SourceStore;
@@ -45,7 +44,7 @@ public class PolicyRunner {
private final ProcessedLedger processedLedger;
/** Full-listing sweep: resolve every source, then reconcile the ledger. */
public SweepOutcome run(Policy policy) {
public List<String> run(Policy policy) {
return run(policy, SweepKind.FULL);
}
@@ -53,10 +52,10 @@ public class PolicyRunner {
* Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
* one failure does not affect the others. No sources means one run with no input (generator
* pipeline). Missing or disabled sources are skipped so one broken reference does not stop the
* rest. Returns the ids of the runs it started plus what the sweep skipped, so a manual trigger
* can report which runs to follow or why nothing ran.
* rest. Returns the ids of the runs it started (empty when sources yielded no work), so a
* manual trigger can report back which runs to follow.
*/
public SweepOutcome run(Policy policy, SweepKind sweep) {
public List<String> run(Policy policy, SweepKind sweep) {
long sweepStart = System.currentTimeMillis();
PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger);
List<String> runIds = new ArrayList<>();
@@ -96,19 +95,13 @@ public class PolicyRunner {
policy.id());
}
}
return context.outcome(runIds);
return runIds;
}
/**
* Run a stored policy on caller-supplied files (e.g. an editor upload), bypassing its sources.
* The supplied documents are still counted against the virtual {@link EditorSource}, scoped to
* the policy's team, so the Sources overview reports the whole team's editor throughput.
*/
/** Run a stored policy on caller-supplied files (e.g. manual upload), bypassing its sources. */
public PolicyRunHandle runWith(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
PolicyRunHandle handle = policyEngine.runPolicy(policy, inputs, listener);
docCounter.record(EditorSource.counterKey(policy.teamId()), inputs.primary().size());
return handle;
return policyEngine.runPolicy(policy, inputs, listener);
}
/** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */
@@ -86,32 +86,4 @@ final class PolicySweep implements ResolveContext {
synchronized Set<String> presentIdentities() {
return Set.copyOf(present);
}
/**
* Summarise the sweep from state already in hand (no extra ledger reads): the prefetched rows
* were loaded before claiming, and successful claims flipped their entries to PROCESSING, so
* what remains DONE or ERROR is exactly what this sweep skipped.
*/
synchronized SweepOutcome outcome(List<String> runIds) {
int alreadyProcessed = 0;
int parked = 0;
int processing = 0;
for (String identity : present) {
ClaimState state = prefetched.get(identity);
if (state == null) {
continue;
}
switch (state.status()) {
case DONE -> alreadyProcessed++;
case ERROR -> parked++;
case PROCESSING, INTERRUPTED -> processing++;
}
}
return new SweepOutcome(
runIds,
present.size(),
alreadyProcessed,
parked,
Math.max(0, processing - runIds.size()));
}
}
@@ -52,19 +52,7 @@ public class PolicyValidator {
InputSpec spec = source.toInputSpec();
inputSourceFor(spec).validate(spec);
}
validateOutput(policy.output());
}
/**
* Validate an output spec against its sink. Must be called on a request thread (caller's
* principal present) so an S3 output's connection is authorization-checked against the caller -
* ad-hoc runs are never persisted and so never hit {@link #validate(Policy)}, and the worker
* thread that later delivers has no principal, so this is their only access gate.
*
* @throws IllegalArgumentException if the type is unknown or the config is invalid/inaccessible
*/
public void validateOutput(OutputSpec output) {
outputSinkFor(output).validate(output);
outputSinkFor(policy.output()).validate(policy.output());
}
private PolicyTrigger triggerFor(TriggerConfig config) {
@@ -1,19 +0,0 @@
package stirling.software.proprietary.policy.engine;
import java.util.List;
/**
* What one policy sweep found and started, so a manual trigger can explain an empty result instead
* of a blanket "nothing to do": how many files the sources listed, how many were skipped because
* they are already processed at their current version, how many are parked by a failed run (not
* retried until they change or history is cleared), and how many are still in flight from an
* earlier sweep. Counts are zero for {@link SweepKind#LIGHT} sweeps, which do not take a full
* listing.
*/
public record SweepOutcome(
List<String> runIds, int filesListed, int alreadyProcessed, int parked, int inFlight) {
public SweepOutcome {
runIds = runIds == null ? List.of() : List.copyOf(runIds);
}
}
@@ -3,7 +3,6 @@ package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import stirling.software.proprietary.policy.model.InputSpec;
@@ -23,12 +22,6 @@ public interface InputSource {
/** Throws {@link IllegalArgumentException} on bad config. Called on save to fail fast. */
default void validate(InputSpec spec) {}
/** Normalise a source's options before persistence (default no-op); must not mutate the arg. */
default Map<String, Object> prepareOptionsForSave(
Map<String, Object> options, boolean isCreate) {
return options;
}
/**
* Resolve the spec into zero or more units of work, each carrying one run's files and a
* completion hook. Empty list means nothing to run right now. Discovery is read-only - files
@@ -1,287 +0,0 @@
package stirling.software.proprietary.policy.input;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.core.io.AbstractResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.s3.S3Config;
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
import stirling.software.proprietary.policy.s3.S3Identities;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
/**
* Reads input files from an Amazon S3 (or S3-compatible) bucket; each listed object is its own unit
* of work, claimed through the {@link ResolveContext} ledger and tracked in place. Identity and
* version gate come from {@link S3Identities}, so the steady-state sweep never downloads content.
* Options: "connectionId" references the stored S3 connection (an {@code IntegrationConfig} owning
* bucket, region, endpoint, and credentials - resolved by {@link S3ConnectionResolver}); "prefix"
* (only keys starting with it are read) and "mode" are per-source, where mode is "consume"
* (default: a processed object is deleted once every policy that claimed it has settled
* successfully and it is still the version that ran; failures stay in place and are not retried
* until they change) or "snapshot" (stateless, every run sees the full set). Keys ending in "/"
* (folder placeholders) and keys with a dot-prefixed path segment are never picked up, mirroring
* the folder source's hidden-file rule.
*/
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class S3InputSource implements InputSource {
private static final String TYPE = "s3";
private final S3ConnectionPool connectionPool;
private final S3ConnectionResolver connectionResolver;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(InputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
/**
* Fails fast at save time: an unknown/disabled/unusable connection, bad config shape, a private
* endpoint without the operator opt-in, or a bucket the connection cannot list.
*/
@Override
public void validate(InputSpec spec) {
S3Config config = connectionResolver.resolve(spec.options());
try {
connectionPool.clientFor(config).listObjectsV2(listRequest(config).maxKeys(1).build());
} catch (SdkException e) {
throw new IllegalArgumentException(
"cannot access s3://"
+ config.bucket()
+ "/"
+ config.prefix()
+ ": "
+ e.getMessage(),
e);
}
}
@Override
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
S3Config config = connectionResolver.resolve(spec.options());
S3Client client = connectionPool.clientFor(config);
// A listing failure propagates so the sweep reads it as "could not list" (which vetoes
// presence cleanup), never as "verifiably no objects".
List<S3Object> objects = listObjects(client, config);
if (config.snapshot()) {
return objects.stream()
.map(
object ->
ResolvedInput.of(
PolicyInputs.of(
List.of(
objectResource(
client, config, object)))))
.toList();
}
ctx.reportPresent(
objects.stream()
.map(object -> S3Identities.identity(config.bucket(), object.key()))
.toList());
List<ResolvedInput> work = new ArrayList<>();
for (S3Object object : objects) {
String identity = S3Identities.identity(config.bucket(), object.key());
String gate = S3Identities.gate(object.eTag(), object.size(), object.lastModified());
if (!ctx.claim(identity, gate, null)) {
continue;
}
work.add(
new ResolvedInput(
PolicyInputs.of(List.of(objectResource(client, config, object))),
success ->
completeConsumed(
ctx,
client,
config,
object.key(),
identity,
gate,
success)));
}
return work;
}
/**
* Settle at the version this run claimed, then remove the object only when it still carries
* that version and every policy that claimed it has settled DONE, mirroring the folder source's
* consensus delete. A failed run settles ERROR and never deletes; the DONE row of an object
* that could not be deleted still stops reprocessing.
*/
private void completeConsumed(
ResolveContext ctx,
S3Client client,
S3Config config,
String key,
String identity,
String claimGate,
boolean success) {
ctx.settle(identity, claimGate, null, success);
if (!success) {
return;
}
try {
HeadObjectResponse head =
client.headObject(
HeadObjectRequest.builder().bucket(config.bucket()).key(key).build());
String currentGate =
S3Identities.gate(head.eTag(), head.contentLength(), head.lastModified());
if (currentGate.equals(claimGate) && ctx.allSettledDone(identity)) {
client.deleteObject(
DeleteObjectRequest.builder().bucket(config.bucket()).key(key).build());
}
} catch (NoSuchKeyException alreadyGone) {
// Removed by the user or a co-watching policy's own consensus delete: nothing to do.
} catch (S3Exception e) {
if (e.statusCode() == 404) {
return;
}
log.warn("Could not remove consumed S3 object {}: {}", identity, e.getMessage());
} catch (SdkException e) {
log.warn("Could not remove consumed S3 object {}: {}", identity, e.getMessage());
}
}
/** Every ingestible object under the configured prefix, across all listing pages. */
private static List<S3Object> listObjects(S3Client client, S3Config config) {
List<S3Object> objects = new ArrayList<>();
String continuationToken = null;
do {
ListObjectsV2Request.Builder request = listRequest(config);
if (continuationToken != null) {
request.continuationToken(continuationToken);
}
ListObjectsV2Response page = client.listObjectsV2(request.build());
for (S3Object object : page.contents()) {
if (ingestible(object)) {
objects.add(object);
}
}
continuationToken = page.nextContinuationToken();
} while (continuationToken != null);
return objects;
}
private static ListObjectsV2Request.Builder listRequest(S3Config config) {
ListObjectsV2Request.Builder request =
ListObjectsV2Request.builder().bucket(config.bucket());
if (!config.prefix().isEmpty()) {
request.prefix(config.prefix());
}
return request;
}
/**
* Folder-placeholder keys (ending "/") and keys with a dot-prefixed segment are skipped, so a
* hidden convention (e.g. a future output sink's staging prefix) is never re-ingested.
*/
private static boolean ingestible(S3Object object) {
String key = object.key();
if (key.isEmpty() || key.endsWith("/")) {
return false;
}
for (String segment : key.split("/")) {
if (segment.startsWith(".")) {
return false;
}
}
return true;
}
private static Resource objectResource(S3Client client, S3Config config, S3Object object) {
return new S3ObjectResource(client, config.bucket(), object);
}
/**
* Streams the object on demand, pinned to the ETag observed at listing time so a run never
* reads a different version than the sweep claimed (a swapped object fails the read with a
* precondition error and the new version is claimed by a later sweep).
*/
private static final class S3ObjectResource extends AbstractResource {
private final S3Client client;
private final String bucket;
private final String key;
private final String eTag;
private final Long size;
private S3ObjectResource(S3Client client, String bucket, S3Object object) {
this.client = client;
this.bucket = bucket;
this.key = object.key();
this.eTag = object.eTag();
this.size = object.size();
}
@Override
public InputStream getInputStream() throws IOException {
GetObjectRequest.Builder request = GetObjectRequest.builder().bucket(bucket).key(key);
if (eTag != null && !eTag.isBlank()) {
request.ifMatch(eTag);
}
try {
return client.getObject(request.build());
} catch (NoSuchKeyException e) {
throw new FileNotFoundException(getDescription() + " no longer exists");
} catch (SdkException e) {
throw new IOException(
"Could not read " + getDescription() + ": " + e.getMessage(), e);
}
}
/** Listed just now; readers get a precise error from {@link #getInputStream} instead. */
@Override
public boolean exists() {
return true;
}
@Override
public long contentLength() {
return size == null ? -1 : size;
}
@Override
public String getFilename() {
return key.substring(key.lastIndexOf('/') + 1);
}
@Override
public String getDescription() {
return "S3 object " + S3Identities.identity(bucket, key);
}
}
}
@@ -1,191 +0,0 @@
package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.ledger.FolderIdentities;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.webhook.WebhookConfig;
import stirling.software.proprietary.policy.webhook.WebhookIds;
import stirling.software.proprietary.policy.webhook.WebhookSpool;
/** Push source: deliveries staged to an S3 connection or the local spool, read via the ledger. */
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class WebhookInputSource implements InputSource {
static final String TYPE = "webhook";
private final WebhookSpool spool;
private final FileReadinessChecker readinessChecker;
// Connection-backed staging delegates to the S3 source; else the local spool.
private final S3InputSource s3InputSource;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(InputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
@Override
public void validate(InputSpec spec) {
WebhookConfig config = WebhookConfig.from(spec.options());
if (config.usesConnection()) {
// Resolve with the caller present so save fails if they can't use the connection.
s3InputSource.validate(stagingSpec(config));
}
}
/** The webhook's durable staging as an {@code s3} input spec for {@link S3InputSource}. */
private static InputSpec stagingSpec(WebhookConfig config) {
return new InputSpec(
"s3",
Map.of(
WebhookConfig.CONNECTION_ID_OPTION,
config.connectionId(),
"prefix",
config.stagingPrefix(),
"mode",
config.mode()));
}
/** Mint the routing id + signing secret on create; an existing webhook is left untouched. */
@Override
public Map<String, Object> prepareOptionsForSave(
Map<String, Object> options, boolean isCreate) {
boolean hasId =
options.get(WebhookConfig.WEBHOOK_ID_OPTION) != null
&& !options.get(WebhookConfig.WEBHOOK_ID_OPTION).toString().isBlank();
if (!isCreate && hasId) {
return options;
}
Map<String, Object> prepared = new LinkedHashMap<>(options);
if (!hasId) {
prepared.put(WebhookConfig.WEBHOOK_ID_OPTION, WebhookIds.newWebhookId());
}
Object secret = prepared.get(WebhookConfig.SIGNING_SECRET_OPTION);
if (secret == null || secret.toString().isBlank()) {
prepared.put(WebhookConfig.SIGNING_SECRET_OPTION, WebhookIds.newSigningSecret());
}
return prepared;
}
@Override
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
WebhookConfig config = WebhookConfig.from(spec.options());
if (config.usesConnection()) {
// Durable staging read by the S3 source (no principal; the save-time check is trusted).
return s3InputSource.resolve(stagingSpec(config), ctx);
}
Path dir = spool.dirFor(config.webhookId());
if (!Files.isDirectory(dir)) {
// No deliveries yet: verifiably empty (a missing dir is normal here, not an error).
ctx.reportPresent(List.of());
return List.of();
}
Path canonicalDir = FolderIdentities.canonicalDir(dir);
List<Path> present = listFiles(dir);
if (config.snapshot()) {
List<ResolvedInput> work = new ArrayList<>();
for (Path file : present) {
if (readinessChecker.isReady(file)) {
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
}
}
return work;
}
ctx.reportPresent(
present.stream()
.map(file -> FolderIdentities.identity(canonicalDir, dir, file))
.toList());
List<ResolvedInput> work = new ArrayList<>();
for (Path file : present) {
if (!readinessChecker.isReady(file)) {
continue;
}
String identity = FolderIdentities.identity(canonicalDir, dir, file);
String gate;
boolean claimed;
try {
gate = FolderIdentities.statGate(file);
claimed = ctx.claim(identity, gate, null);
} catch (IOException | UncheckedIOException e) {
log.debug("Could not read {} for its version: {}", file, e.getMessage());
continue; // vanished or unreadable mid-sweep; the next sweep sees the truth
}
if (!claimed) {
continue;
}
work.add(
new ResolvedInput(
PolicyInputs.of(List.of(fileResource(file))),
success -> completeConsumed(ctx, identity, file, gate, success)));
}
return work;
}
/** Settle, then delete when unchanged and every claimant settled DONE (consensus). */
private static void completeConsumed(
ResolveContext ctx, String identity, Path file, String claimGate, boolean success) {
ctx.settle(identity, claimGate, null, success);
if (!success) {
return;
}
try {
if (FolderIdentities.statGate(file).equals(claimGate) && ctx.allSettledDone(identity)) {
Files.deleteIfExists(file);
}
} catch (java.nio.file.NoSuchFileException alreadyGone) {
// Removed by the user or a co-watching policy's own consensus delete: nothing to do.
} catch (IOException e) {
log.warn("Could not remove consumed webhook delivery {}: {}", file, e.getMessage());
}
}
/** Every non-hidden regular file currently spooled for the webhook. */
private static List<Path> listFiles(Path dir) throws IOException {
List<Path> files = new ArrayList<>();
try (Stream<Path> entries = Files.list(dir)) {
entries.filter(Files::isRegularFile)
.filter(file -> !file.getFileName().toString().startsWith("."))
.forEach(files::add);
}
return files;
}
private static Resource fileResource(Path path) {
String name = WebhookSpool.displayName(path.getFileName().toString());
return new FileSystemResource(path.toFile()) {
@Override
public String getFilename() {
return name;
}
};
}
}
@@ -15,6 +15,7 @@ import java.util.List;
import java.util.UUID;
import java.util.stream.Stream;
import org.apache.commons.io.FilenameUtils;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
@@ -81,7 +82,7 @@ public class FolderOutputSink implements PolicyOutputSink {
List<ResultFile> results = new ArrayList<>();
for (int i = 0; i < outputs.size(); i++) {
Resource resource = outputs.get(i);
String name = OutputNames.safeName(resource.getFilename(), i);
String name = safeName(resource.getFilename(), i);
Path staged = tmpDir.resolve(UUID.randomUUID().toString());
String contentHash = stage(resource, staged, delivery.policyId() != null);
long size = Files.size(staged);
@@ -197,14 +198,29 @@ public class FolderOutputSink implements PolicyOutputSink {
return Path.of(directory.toString());
}
// Strip any directory component / "../" so a crafted output name cannot escape targetDir.
private static String safeName(String filename, int index) {
if (filename == null || filename.isBlank()) {
return "output-" + index;
}
String name = FilenameUtils.getName(filename);
if (name.isBlank() || ".".equals(name) || "..".equals(name)) {
return "output-" + index;
}
return name;
}
// Non-colliding path, appending " (n)" before the extension.
private static Path uniqueTarget(Path dir, String filename) {
Path candidate = dir.resolve(filename);
if (!Files.exists(candidate)) {
return candidate;
}
String base = FilenameUtils.getBaseName(filename);
String ext = FilenameUtils.getExtension(filename);
String suffix = ext.isEmpty() ? "" : "." + ext;
for (int n = 1; ; n++) {
Path next = dir.resolve(OutputNames.numbered(filename, n));
Path next = dir.resolve(base + " (" + n + ")" + suffix);
if (!Files.exists(next)) {
return next;
}
@@ -1,29 +0,0 @@
package stirling.software.proprietary.policy.output;
import org.apache.commons.io.FilenameUtils;
/** Output file naming shared by the sinks: sanitised base names and collision suffixes. */
final class OutputNames {
private OutputNames() {}
/** Strip any directory component / "../" so a crafted output name cannot escape the target. */
static String safeName(String filename, int index) {
if (filename == null || filename.isBlank()) {
return "output-" + index;
}
String name = FilenameUtils.getName(filename);
if (name.isBlank() || ".".equals(name) || "..".equals(name)) {
return "output-" + index;
}
return name;
}
/** The nth alternative for a taken name, appending " (n)" before the extension. */
static String numbered(String filename, int n) {
String base = FilenameUtils.getBaseName(filename);
String ext = FilenameUtils.getExtension(filename);
String suffix = ext.isEmpty() ? "" : "." + ext;
return base + " (" + n + ")" + suffix;
}
}
@@ -1,272 +0,0 @@
package stirling.software.proprietary.policy.output;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HexFormat;
import java.util.List;
import java.util.UUID;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.MediaTypeFactory;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.s3.S3Config;
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
import stirling.software.proprietary.policy.s3.S3Identities;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
/**
* Uploads a run's outputs to the bucket and key prefix given in the {@link OutputSpec} (same
* connection options as the S3 input source; "prefix" is the destination folder). The
* record-before-visible obligation is met without a rename step: a single-part PUT's ETag is the
* MD5 of its content on plain and SSE-S3 buckets, so the ledger row is recorded at that predicted
* gate BEFORE the upload, and the object is claimed under exactly the gate the next listing
* returns. Stores where the returned ETag differs (e.g. SSE-KMS) are re-recorded at the actual gate
* immediately after the PUT - a narrow race those buckets accept rather than a broken loop. Names
* never overwrite: uploads are conditional on the key not existing ({@code If-None-Match: *}),
* re-picking "name (n).ext" on collision exactly like the folder sink; stores without
* conditional-write support fall back to an existence check per candidate.
*/
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class S3OutputSink implements PolicyOutputSink {
private static final String TYPE = "s3";
private final S3ConnectionPool connectionPool;
private final S3ConnectionResolver connectionResolver;
private final ProcessedLedger processedLedger;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(OutputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
/**
* Connection resolution (including the saving user's right to use it) and endpoint guard only -
* no network probe, since write-only credentials (s3:PutObject without s3:ListBucket) are a
* legitimate setup for an output bucket and a listing probe would wrongly reject them.
*/
@Override
public void validate(OutputSpec spec) {
connectionPool.clientFor(connectionResolver.resolve(spec.options()));
}
@Override
public List<ResultFile> deliver(
OutputDelivery delivery, List<Resource> outputs, OutputSpec spec) throws IOException {
S3Config config = connectionResolver.resolve(spec.options());
S3Client client = connectionPool.clientFor(config);
List<ResultFile> results = new ArrayList<>();
for (int i = 0; i < outputs.size(); i++) {
Resource resource = outputs.get(i);
String name = OutputNames.safeName(resource.getFilename(), i);
Path staged = Files.createTempFile("s3-output-", ".tmp");
try {
String predictedGate = stage(resource, staged, delivery.policyId() != null);
long size = Files.size(staged);
String key = upload(delivery, client, config, name, staged, predictedGate);
String contentType =
MediaTypeFactory.getMediaType(name)
.orElse(MediaType.APPLICATION_OCTET_STREAM)
.toString();
results.add(
ResultFile.builder()
.fileId(UUID.randomUUID().toString())
.fileName(S3Identities.identity(config.bucket(), key))
.contentType(contentType)
.fileSize(size)
.build());
log.debug(
"Wrote policy run {} output to {}",
delivery.runId(),
S3Identities.identity(config.bucket(), key));
} finally {
try {
Files.deleteIfExists(staged);
} catch (IOException e) {
log.warn("Could not remove S3 staging file {}: {}", staged, e.getMessage());
}
}
}
return results;
}
/**
* Spool the output to a local staging file (S3 needs a known content length, and the body must
* be re-readable across collision retries). For a recorded delivery the MD5 - the predicted
* single-part ETag - is digested in the same pass; ad-hoc runs record nothing and skip it.
*/
private static String stage(Resource resource, Path staged, boolean recorded)
throws IOException {
if (!recorded) {
try (InputStream is = resource.getInputStream();
OutputStream out = Files.newOutputStream(staged)) {
is.transferTo(out);
}
return null;
}
MessageDigest digest = newMd5();
try (InputStream is = resource.getInputStream();
DigestOutputStream out =
new DigestOutputStream(Files.newOutputStream(staged), digest)) {
is.transferTo(out);
}
return HexFormat.of().formatHex(digest.digest());
}
/**
* The S3 shape of the folder sink's record-then-rename loop. The ledger row must exist before
* the object is visible, so it is recorded at the predicted gate before the PUT; losing the
* chosen key to a concurrent writer (the conditional PUT fails) forgets the just-recorded row -
* whatever object actually owns that key must stay claimable at any version - then re-picks. A
* PUT that never made the object visible also forgets its row.
*/
private String upload(
OutputDelivery delivery,
S3Client client,
S3Config config,
String name,
Path staged,
String predictedGate)
throws IOException {
String keyPrefix = keyPrefix(config);
boolean conditionalPuts = true;
for (int attempt = 0; ; attempt++) {
String key = keyPrefix + (attempt == 0 ? name : OutputNames.numbered(name, attempt));
String identity = S3Identities.identity(config.bucket(), key);
if (!conditionalPuts && exists(client, config.bucket(), key)) {
continue;
}
if (delivery.policyId() != null) {
processedLedger.recordOutput(delivery.policyId(), identity, predictedGate, null);
}
PutObjectRequest.Builder put =
PutObjectRequest.builder().bucket(config.bucket()).key(key);
if (conditionalPuts) {
put.ifNoneMatch("*");
}
try {
PutObjectResponse response =
client.putObject(put.build(), RequestBody.fromFile(staged));
reRecordIfGateDiffers(delivery, identity, predictedGate, response);
return key;
} catch (S3Exception e) {
forgetRecorded(delivery, identity, predictedGate);
if (conditionalPuts && e.statusCode() == 412) {
// Known edge: if our own PUT succeeded server-side but the response was lost
// and the SDK retried, that retry 412s here too - we then upload under the
// next name, leaving the first object row-less (claimable, single duplicate).
// Requires a response-lost network flake at exactly this moment; accepted.
log.debug("Output key {} taken concurrently; re-picking", identity);
continue;
}
if (conditionalPuts && e.statusCode() == 501) {
// Store without conditional-write support: retry this candidate with a plain
// existence check instead.
log.debug(
"Conditional PUT unsupported by {}; falling back to existence checks",
config.bucket());
conditionalPuts = false;
attempt--;
continue;
}
throw new IOException("Could not upload " + identity + ": " + e.getMessage(), e);
} catch (SdkException e) {
forgetRecorded(delivery, identity, predictedGate);
throw new IOException("Could not upload " + identity + ": " + e.getMessage(), e);
}
}
}
/**
* On buckets where a PUT's ETag is not the content MD5 (e.g. SSE-KMS), re-record at the gate
* listings will actually return. The row is briefly at the wrong gate while the object is
* already visible - the narrow race such stores trade for a working self-output skip.
*/
private void reRecordIfGateDiffers(
OutputDelivery delivery,
String identity,
String predictedGate,
PutObjectResponse response) {
if (delivery.policyId() == null) {
return;
}
String actualGate = S3Identities.gate(response.eTag(), null, null);
if (!actualGate.equals(predictedGate)) {
log.debug(
"PUT ETag for {} differs from content MD5 (encrypted bucket?); re-recording",
identity);
processedLedger.recordOutput(delivery.policyId(), identity, actualGate, null);
}
}
private void forgetRecorded(OutputDelivery delivery, String identity, String predictedGate) {
if (delivery.policyId() != null) {
processedLedger.forgetOutput(delivery.policyId(), identity, predictedGate);
}
}
private static boolean exists(S3Client client, String bucket, String key) {
try {
client.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build());
return true;
} catch (NoSuchKeyException e) {
return false;
} catch (S3Exception e) {
if (e.statusCode() == 404) {
return false;
}
throw e;
}
}
/** The configured prefix as a key-path prefix: "processed" and "processed/" mean the same. */
private static String keyPrefix(S3Config config) {
String prefix = config.prefix();
if (prefix.isEmpty() || prefix.endsWith("/")) {
return prefix;
}
return prefix + "/";
}
private static MessageDigest newMd5() {
try {
return MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("MD5 unavailable", e);
}
}
}
@@ -1,212 +0,0 @@
package stirling.software.proprietary.policy.s3;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.access.model.DefaultAccessPolicy;
import stirling.software.proprietary.access.model.OwnerScope;
import stirling.software.proprietary.integration.model.IntegrationConfig;
import stirling.software.proprietary.integration.model.IntegrationType;
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
import stirling.software.proprietary.model.Team;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.security.repository.TeamRepository;
import tools.jackson.databind.ObjectMapper;
/**
* One-time, idempotent extraction of legacy embedded S3 credentials into stored connections:
* sources and policy outputs written before connections shipped carry bucket/credentials in their
* own options; this rewrites each to reference a (deduplicated) S3 {@link IntegrationConfig} and
* keeps only per-use options (prefix, mode). MUST be programmatic - the option JSON is encrypted at
* the application layer, so no SQL migration can read it.
*
* <p>Idempotent by construction: rewritten rows no longer embed credentials, so re-runs find
* nothing to do. Connections are deduplicated against both this run's extractions and existing S3
* connections; a concurrent multi-node boot can at worst create a redundant connection row, never
* corrupt a source. Ownership follows the owning row: team-scoped when the source/policy has a
* team, server-scoped otherwise (single-operator self-hosted).
*/
@Slf4j
@Component
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class EmbeddedS3CredentialMigration {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final List<String> CONNECTION_OPTIONS =
List.of("bucket", "region", "endpoint", "accessKeyId", "secretAccessKey");
// Field separator for the dedup key: a unit-separator control char that cannot appear in a
// bucket/region/endpoint/credential, so distinct field sets can never collide.
private static final char DELIMITER = '\u001f';
private final SourceStore sourceStore;
private final PolicyStore policyStore;
private final IntegrationConfigRepository connections;
private final TeamRepository teamRepository;
@EventListener(ApplicationReadyEvent.class)
@Transactional
public void migrate() {
Map<String, IntegrationConfig> byCredentialKey = indexExistingConnections();
int migrated = 0;
for (Source source : sourceStore.all()) {
if (!"s3".equals(source.type()) || !embedsCredentials(source.options())) {
continue;
}
IntegrationConfig connection =
connectionFor(source.options(), source.teamId(), byCredentialKey);
sourceStore.save(withOptions(source, referencing(connection, source.options(), true)));
migrated++;
}
for (Policy policy : policyStore.all()) {
OutputSpec output = policy.output();
if (!"s3".equals(output.type()) || !embedsCredentials(output.options())) {
continue;
}
IntegrationConfig connection =
connectionFor(output.options(), policy.teamId(), byCredentialKey);
policyStore.save(
withOutput(
policy,
new OutputSpec(
output.type(),
referencing(connection, output.options(), false))));
migrated++;
}
if (migrated > 0) {
log.info("Extracted embedded S3 credentials from {} row(s) into connections", migrated);
}
}
private static boolean embedsCredentials(Map<String, Object> options) {
return options.get("accessKeyId") != null;
}
/** Reuses an existing connection with identical coordinates+credentials, else creates one. */
private IntegrationConfig connectionFor(
Map<String, Object> options, Long teamId, Map<String, IntegrationConfig> byKey) {
String key = credentialKey(options);
IntegrationConfig existing = byKey.get(key);
if (existing != null) {
return existing;
}
IntegrationConfig connection = new IntegrationConfig();
connection.setIntegrationType(IntegrationType.S3);
connection.setName(connectionName(options, byKey));
connection.setEnabled(true);
connection.setLocked(false);
connection.setDefaultAccess(DefaultAccessPolicy.EXPLICIT_ONLY);
Team team = teamId == null ? null : teamRepository.findById(teamId).orElse(null);
if (team != null) {
connection.setScope(OwnerScope.TEAM);
connection.setOwnerTeam(team);
} else {
// No team (teamless self-hosted, or a source whose team was since deleted): server
// scope, i.e. admin-owned. An orphaned-team source's non-admin editor would then need
// an admin to re-share the connection - acceptable for the narrow orphaned case.
connection.setScope(OwnerScope.SERVER);
}
Map<String, Object> config = new LinkedHashMap<>();
for (String option : CONNECTION_OPTIONS) {
Object value = options.get(option);
if (value != null && !value.toString().isBlank()) {
config.put(option, value);
}
}
connection.setConfig(OBJECT_MAPPER.writeValueAsString(config));
IntegrationConfig saved = connections.save(connection);
byKey.put(key, saved);
return saved;
}
/** The rewritten options: the connection reference plus per-use settings only. */
private static Map<String, Object> referencing(
IntegrationConfig connection, Map<String, Object> legacy, boolean keepMode) {
Map<String, Object> options = new LinkedHashMap<>();
options.put(S3ConnectionResolver.CONNECTION_ID_OPTION, connection.getId());
Object prefix = legacy.get("prefix");
if (prefix != null && !prefix.toString().isBlank()) {
options.put("prefix", prefix);
}
Object mode = legacy.get("mode");
if (keepMode && mode != null && !mode.toString().isBlank()) {
options.put("mode", mode);
}
return options;
}
private Map<String, IntegrationConfig> indexExistingConnections() {
Map<String, IntegrationConfig> byKey = new LinkedHashMap<>();
for (IntegrationConfig connection : connections.findAll()) {
if (connection.getIntegrationType() != IntegrationType.S3) {
continue;
}
try {
Map<String, Object> config =
OBJECT_MAPPER.readValue(connection.getConfig(), Map.class);
byKey.putIfAbsent(credentialKey(config), connection);
} catch (Exception e) {
log.debug(
"Skipping unreadable S3 connection {} while indexing: {}",
connection.getId(),
e.getMessage());
}
}
return byKey;
}
private static String credentialKey(Map<String, Object> options) {
StringBuilder key = new StringBuilder();
for (String option : CONNECTION_OPTIONS) {
Object value = options.get(option);
key.append(value == null ? "" : value.toString().trim()).append(DELIMITER);
}
return key.toString();
}
private static String connectionName(
Map<String, Object> options, Map<String, IntegrationConfig> byKey) {
String base = "S3: " + options.getOrDefault("bucket", "bucket");
long sameName = byKey.values().stream().filter(c -> c.getName().startsWith(base)).count();
return sameName == 0 ? base : base + " (" + (sameName + 1) + ")";
}
private static Source withOptions(Source source, Map<String, Object> options) {
return new Source(
source.id(),
source.name(),
source.type(),
options,
source.enabled(),
source.owner(),
source.teamId());
}
private static Policy withOutput(Policy policy, OutputSpec output) {
return new Policy(
policy.id(),
policy.name(),
policy.owner(),
policy.enabled(),
policy.trigger(),
policy.sourceIds(),
policy.steps(),
output,
policy.teamId());
}
}
@@ -1,56 +0,0 @@
package stirling.software.proprietary.policy.s3;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.integration.service.IntegrationConfigUsageCheck;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.store.PolicyStore;
/**
* Reports the policy sources and pipeline outputs referencing an S3 connection, so the connection
* cannot be deleted out from under them (mirrors {@code SourceController}'s referenced-source
* delete guard). Scans in memory - fine at admin-dashboard scale, always consistent with the live
* stores.
*/
@Component
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class PolicyS3ConnectionUsageCheck implements IntegrationConfigUsageCheck {
private final SourceStore sourceStore;
private final PolicyStore policyStore;
@Override
public List<String> usagesOf(long configId) {
List<String> usages = new ArrayList<>();
for (Source source : sourceStore.all()) {
if (references(source.options(), configId)) {
usages.add("source '" + source.name() + "'");
}
}
for (Policy policy : policyStore.all()) {
if (references(policy.output().options(), configId)) {
usages.add("pipeline '" + policy.name() + "'");
}
}
return usages;
}
private static boolean references(Map<String, Object> options, long configId) {
try {
Long reference = S3ConnectionResolver.connectionId(options);
return reference != null && reference == configId;
} catch (IllegalArgumentException unparseable) {
return false;
}
}
}

Some files were not shown because too many files have changed in this diff Show More