Automations dispatch each tool step as a separate internal loopback POST via {@link
+ * InternalApiClient}. The orchestrator opens a run scope around its dispatch loop; {@code
+ * InternalApiClient} reads {@link #current()} and stamps it on every sub-step request as {@link
+ * #RUN_ID_HEADER}. The SaaS PAYG interceptor uses that header so all sub-steps of ONE run group
+ * into a single charge, while two separate runs that happen to touch identical bytes stay
+ * distinct charges (the old content+time-window grouping merged them).
+ *
+ *
Sub-steps dispatch synchronously on the orchestrator's own thread (loopback {@code
+ * RestTemplate}), so this ThreadLocal is visible to {@code InternalApiClient}. The id then crosses
+ * to the receiving request thread via the HTTP header — never via this ThreadLocal.
+ *
+ *
No-op when the id is absent (a standalone tool call): the interceptor treats a missing run id
+ * as "its own charge", which is exactly what a one-off call should be.
+ */
+public final class AutomationRunContext {
+
+ /** Header carrying the run id on internal sub-step dispatches. */
+ public static final String RUN_ID_HEADER = "X-Stirling-Run-Id";
+
+ private static final ThreadLocal CURRENT = new ThreadLocal<>();
+
+ private AutomationRunContext() {}
+
+ /**
+ * Opens a run scope on the current thread. Returns an {@link AutoCloseable} that restores the
+ * previously-active id (nesting-safe) — use in try-with-resources around the dispatch loop.
+ */
+ public static Scope open(String runId) {
+ String previous = CURRENT.get();
+ CURRENT.set(runId);
+ return () -> {
+ if (previous == null) {
+ CURRENT.remove();
+ } else {
+ CURRENT.set(previous);
+ }
+ };
+ }
+
+ /** The run id active on this thread, or {@code null} when not inside a run scope. */
+ public static String current() {
+ return CURRENT.get();
+ }
+
+ /** AutoCloseable whose {@link #close()} declares no checked exception. */
+ public interface Scope extends AutoCloseable {
+ @Override
+ void close();
+ }
+}
diff --git a/app/common/src/main/java/stirling/software/common/service/InternalApiClient.java b/app/common/src/main/java/stirling/software/common/service/InternalApiClient.java
index 8df2d5e410..63d577ca85 100644
--- a/app/common/src/main/java/stirling/software/common/service/InternalApiClient.java
+++ b/app/common/src/main/java/stirling/software/common/service/InternalApiClient.java
@@ -8,6 +8,7 @@ import java.nio.file.Files;
import java.time.Duration;
import java.util.regex.Pattern;
+import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.core.io.FileSystemResource;
@@ -60,6 +61,17 @@ public class InternalApiClient {
*/
public static final String AUTOMATION_HEADER = "X-Stirling-Automation";
+ /**
+ * Header carrying the parent policy's name onto each sub-step dispatch, read from MDC key
+ * {@link #POLICY_NAME_MDC_KEY} (set by the policy runner on the worker thread). Lets the audit
+ * layer attribute a tool step to the policy that ran it, instead of showing it as a bare direct
+ * call.
+ */
+ public static final String POLICY_NAME_HEADER = "X-Stirling-Policy-Name";
+
+ /** MDC key the policy runner stamps with the running policy's name; forwarded as a header. */
+ public static final String POLICY_NAME_MDC_KEY = "auditPolicyName";
+
private final ServletContext servletContext;
private final UserServiceInterface userService;
private final TempFileManager tempFileManager;
@@ -111,6 +123,27 @@ public class InternalApiClient {
// step inside a policy run must bill as AUTOMATION, not AI). Set unconditionally because
// every caller of this dispatcher is an automation surface by design.
headers.add(AUTOMATION_HEADER, "true");
+ // Propagate the current automation run id (set by the orchestrator around its dispatch
+ // loop) so the PAYG interceptor groups every sub-step of this one run into a single charge,
+ // and never merges two separate runs that happen to touch identical bytes. Absent → the
+ // receiving call is treated as standalone. See AutomationRunContext.
+ String runId = AutomationRunContext.current();
+ if (runId != null && !runId.isEmpty()) {
+ headers.add(AutomationRunContext.RUN_ID_HEADER, runId);
+ }
+
+ // Forward the parent policy name (set in MDC by the policy runner) so the audited sub-step
+ // ties back to its policy. Single-line, length-capped: it becomes an HTTP header value.
+ String policyName = MDC.get(POLICY_NAME_MDC_KEY);
+ if (policyName != null && !policyName.isBlank()) {
+ String safe = policyName.replaceAll("[\\r\\n]", " ").trim();
+ if (safe.length() > 200) {
+ safe = safe.substring(0, 200);
+ }
+ if (!safe.isEmpty()) {
+ headers.add(POLICY_NAME_HEADER, safe);
+ }
+ }
// A no-file ai/tools call (e.g. create-pdf-from-html-agent) sends only string params, so
// without this RestTemplate would use urlencoded instead of the multipart the controller
diff --git a/app/common/src/main/java/stirling/software/common/service/TempFileCleanupService.java b/app/common/src/main/java/stirling/software/common/service/TempFileCleanupService.java
index 81b45aed09..438481c212 100644
--- a/app/common/src/main/java/stirling/software/common/service/TempFileCleanupService.java
+++ b/app/common/src/main/java/stirling/software/common/service/TempFileCleanupService.java
@@ -144,8 +144,10 @@ public class TempFileCleanupService {
int directoriesDeletedCount = 0;
for (Path directory : registry.getTempDirectories()) {
try {
- if (Files.exists(directory)) {
+ if (Files.exists(directory)
+ && shouldDeleteRegisteredDirectory(directory, maxAgeMillis)) {
GeneralUtils.deleteDirectory(directory);
+ registry.unregisterDirectory(directory);
directoriesDeletedCount++;
log.debug("Cleaned up temporary directory: {}", directory);
}
@@ -275,6 +277,21 @@ public class TempFileCleanupService {
return totalDeletedCount.get();
}
+ private boolean shouldDeleteRegisteredDirectory(Path directory, long maxAgeMillis) {
+ if (maxAgeMillis <= 0) {
+ return true;
+ }
+
+ try {
+ long currentTime = System.currentTimeMillis();
+ long lastModified = Files.getLastModifiedTime(directory).toMillis();
+ return (currentTime - lastModified) > maxAgeMillis;
+ } catch (IOException e) {
+ log.debug("Could not check directory age, skipping cleanup: {}", directory, e);
+ return false;
+ }
+ }
+
/** Get the system temp directory path based on configuration or system property. */
private Path getSystemTempPath() {
String systemTempDir =
diff --git a/app/common/src/main/java/stirling/software/common/util/TempFileManager.java b/app/common/src/main/java/stirling/software/common/util/TempFileManager.java
index ff31975837..0d977e7d06 100644
--- a/app/common/src/main/java/stirling/software/common/util/TempFileManager.java
+++ b/app/common/src/main/java/stirling/software/common/util/TempFileManager.java
@@ -155,6 +155,7 @@ public class TempFileManager {
if (directory != null && Files.isDirectory(directory)) {
try {
GeneralUtils.deleteDirectory(directory);
+ registry.unregisterDirectory(directory);
log.debug("Deleted temp directory: {}", directory.toString());
} catch (IOException e) {
log.warn("Failed to delete temp directory: {}", directory.toString(), e);
diff --git a/app/common/src/main/java/stirling/software/common/util/TempFileRegistry.java b/app/common/src/main/java/stirling/software/common/util/TempFileRegistry.java
index ab1304f5e4..84a3c12272 100644
--- a/app/common/src/main/java/stirling/software/common/util/TempFileRegistry.java
+++ b/app/common/src/main/java/stirling/software/common/util/TempFileRegistry.java
@@ -85,6 +85,18 @@ public class TempFileRegistry {
return directory;
}
+ /**
+ * Unregister a temporary directory from the registry.
+ *
+ * @param directory The directory to unregister
+ */
+ public void unregisterDirectory(Path directory) {
+ if (directory != null) {
+ tempDirectories.remove(directory);
+ log.debug("Unregistered temp directory: {}", directory.toString());
+ }
+ }
+
/**
* Register a third-party temporary file that requires special handling.
*
diff --git a/app/common/src/test/java/stirling/software/common/service/TempFileCleanupServiceMoreTest.java b/app/common/src/test/java/stirling/software/common/service/TempFileCleanupServiceMoreTest.java
index 802dc2eace..cfbe950574 100644
--- a/app/common/src/test/java/stirling/software/common/service/TempFileCleanupServiceMoreTest.java
+++ b/app/common/src/test/java/stirling/software/common/service/TempFileCleanupServiceMoreTest.java
@@ -176,11 +176,13 @@ class TempFileCleanupServiceMoreTest {
class ScheduledCleanup {
@Test
- @DisplayName("deletes registered temp directories and reports counts")
+ @DisplayName("deletes stale registered temp directories and reports counts")
void deletesRegisteredDirectories() throws IOException {
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(2);
Path regDir = Files.createDirectories(tempDir.resolve("registeredDir"));
Files.createFile(regDir.resolve("inside.txt"));
+ Files.setLastModifiedTime(
+ regDir, FileTime.fromMillis(System.currentTimeMillis() - 2L * 60 * 60 * 1000));
Set dirs = new HashSet<>();
dirs.add(regDir);
when(registry.getTempDirectories()).thenReturn(dirs);
@@ -193,6 +195,22 @@ class TempFileCleanupServiceMoreTest {
verify(tempFileManager).cleanupOldTempFiles(anyLong());
}
+ @Test
+ @DisplayName("keeps a fresh registered temp directory")
+ void keepsFreshRegisteredDirectory() throws IOException {
+ when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(0);
+ Path regDir = Files.createDirectories(tempDir.resolve("freshRegisteredDir"));
+ Files.createFile(regDir.resolve("inside.txt"));
+ Set dirs = new HashSet<>();
+ dirs.add(regDir);
+ when(registry.getTempDirectories()).thenReturn(dirs);
+ lenient().when(registry.contains(any(File.class))).thenReturn(false);
+
+ withIsolatedUserHome(cleanupService::scheduledCleanup);
+
+ assertThat(Files.exists(regDir)).isTrue();
+ }
+
@Test
@DisplayName("skips a registered directory that no longer exists")
void skipsMissingRegisteredDirectory() {
diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java
index c57a43f27f..c4f1c76f5e 100644
--- a/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java
+++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java
@@ -10,6 +10,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.UUID;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
@@ -27,6 +28,7 @@ import stirling.software.SPDF.model.PipelineConfig;
import stirling.software.SPDF.model.PipelineOperation;
import stirling.software.SPDF.model.PipelineResult;
import stirling.software.SPDF.service.ApiDocService;
+import stirling.software.common.service.AutomationRunContext;
import stirling.software.common.service.InternalApiClient;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.ZipExtractionUtils;
@@ -71,6 +73,17 @@ public class PipelineProcessor {
PipelineResult runPipelineAgainstFiles(List outputFiles, PipelineConfig config)
throws Exception {
+ // One pipeline execution = one automation run. Scope a run id so every tool sub-step
+ // dispatched via InternalApiClient groups into a single charge on the SaaS billing side
+ // (see AutomationRunContext); pipeline steps run synchronously on this thread.
+ try (AutomationRunContext.Scope ignored =
+ AutomationRunContext.open(UUID.randomUUID().toString())) {
+ return runPipelineAgainstFilesInternal(outputFiles, config);
+ }
+ }
+
+ private PipelineResult runPipelineAgainstFilesInternal(
+ List outputFiles, PipelineConfig config) throws Exception {
PipelineResult result = new PipelineResult();
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java
index 9c9bce3bd2..be2d10e6e2 100644
--- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java
+++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java
@@ -133,11 +133,17 @@ public class RedactController {
+ "Users can provide text patterns to redact, with options for regex and whole word matching. "
+ "Input:PDF Output:PDF Type:SISO")
public ResponseEntity redactPdf(@ModelAttribute RedactPdfRequest request) {
- String[] listOfText = request.getListOfText().split("\n");
+ String rawListOfText = request.getListOfText();
boolean useRegex = Boolean.TRUE.equals(request.getUseRegex());
boolean wholeWordSearchBool = Boolean.TRUE.equals(request.getWholeWordSearch());
- if (listOfText.length == 0 || (listOfText.length == 1 && listOfText[0].trim().isEmpty())) {
+ if (rawListOfText == null || rawListOfText.trim().isEmpty()) {
+ throw ExceptionUtils.createIllegalArgumentException(
+ "error.redaction.no.patterns", "No text patterns provided for redaction");
+ }
+
+ String[] listOfText = rawListOfText.split("\n");
+ if (listOfText.length == 1 && listOfText[0].trim().isEmpty()) {
throw ExceptionUtils.createIllegalArgumentException(
"error.redaction.no.patterns", "No text patterns provided for redaction");
}
diff --git a/app/core/src/main/resources/static/api-wordmark.svg b/app/core/src/main/resources/static/api-wordmark.svg
index deac1ee165..3658102f90 100644
--- a/app/core/src/main/resources/static/api-wordmark.svg
+++ b/app/core/src/main/resources/static/api-wordmark.svg
@@ -1,4 +1 @@
-
+
\ No newline at end of file
diff --git a/app/core/src/main/resources/static/images/redact-auto.svg b/app/core/src/main/resources/static/images/redact-auto.svg
index 5c87b738ef..7c29b0fc4b 100644
--- a/app/core/src/main/resources/static/images/redact-auto.svg
+++ b/app/core/src/main/resources/static/images/redact-auto.svg
@@ -1,18 +1 @@
-
-
+
\ No newline at end of file
diff --git a/app/core/src/main/resources/static/images/rename.svg b/app/core/src/main/resources/static/images/rename.svg
index 6b19bec208..97946a086d 100644
--- a/app/core/src/main/resources/static/images/rename.svg
+++ b/app/core/src/main/resources/static/images/rename.svg
@@ -1,13 +1 @@
-
-
-
+
\ No newline at end of file
diff --git a/app/core/src/main/resources/static/images/stirling-logo-white.png b/app/core/src/main/resources/static/images/stirling-logo-white.png
index 8018b0771c..c66dcbf3f0 100644
Binary files a/app/core/src/main/resources/static/images/stirling-logo-white.png and b/app/core/src/main/resources/static/images/stirling-logo-white.png differ
diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerMoreTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerMoreTest.java
index 6eafecc1a5..150a2355de 100644
--- a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerMoreTest.java
+++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerMoreTest.java
@@ -299,6 +299,18 @@ class RedactControllerMoreTest {
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
}
+ @Test
+ @DisplayName("null listOfText throws an illegal-argument error before any load")
+ void nullPatternsThrows() throws Exception {
+ RedactPdfRequest request = new RedactPdfRequest();
+ request.setFileInput(pdfFile(new byte[] {1, 2, 3}));
+ request.setListOfText(null);
+
+ assertThatThrownBy(() -> controller.redactPdf(request))
+ .isInstanceOf(RuntimeException.class);
+ verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
+ }
+
@Test
@DisplayName("null file input is reported as a failure")
void nullFileThrows() {
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditAspect.java b/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditAspect.java
index f42a0f29c1..12b2787b75 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditAspect.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditAspect.java
@@ -142,6 +142,9 @@ public class AuditAspect {
auditService.addTimingData(
auditData, startTime, resp, auditedAnnotation.level(), isHttpRequest);
+ // Merge controller-set policy context + the internal-automation marker onto the event.
+ auditService.addAutomationContext(auditData, req);
+
// Resolve the event type based on annotation and context
String httpMethod = null;
String path = null;
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditContext.java b/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditContext.java
new file mode 100644
index 0000000000..650727f5ef
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/audit/AuditContext.java
@@ -0,0 +1,18 @@
+package stirling.software.proprietary.audit;
+
+/**
+ * Request-scoped keys a controller can set to enrich its own audit event with context the generic
+ * aspect can't infer from the HTTP request alone (e.g. the policy a pipeline run belongs to). The
+ * aspect copies these into the audit data in its {@code finally} block, after the controller body
+ * has run. See {@code AuditService#addAutomationContext}.
+ */
+public final class AuditContext {
+
+ /** Request attribute: the name of the policy/pipeline a run executes. */
+ public static final String REQ_ATTR_POLICY_NAME = "stirling.audit.policyName";
+
+ /** Request attribute: the ordered tool endpoint paths a run executes. */
+ public static final String REQ_ATTR_POLICY_STEPS = "stirling.audit.policySteps";
+
+ private AuditContext() {}
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java b/app/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java
index c59c6f3721..2f75b755e0 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java
@@ -205,6 +205,10 @@ public class ControllerAuditAspect {
// Call auditService but with isHttpRequest=true to skip additional timing
auditService.addTimingData(data, start, resp, level, true);
+ // Merge controller-set policy context + the internal-automation marker (set after
+ // the body ran, so it must happen here rather than with the pre-proceed HTTP data).
+ auditService.addAutomationContext(data, req);
+
// Resolve the event type using the unified method
AuditEventType eventType =
auditService.resolveEventType(
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java
index eacdaf32c0..d3aab54c76 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java
@@ -133,29 +133,45 @@ public final class S3Clients {
* storage.s3.allow-private-endpoints=true}.
*/
static void validateEndpointHost(URI endpoint, boolean allowPrivate) {
+ validateEndpointHost(
+ endpoint,
+ allowPrivate,
+ "storage.s3.endpoint",
+ "set storage.s3.allow-private-endpoints=true to opt in"
+ + " (e.g. for MinIO or in-cluster S3).");
+ }
+
+ /**
+ * The same private-address guard for S3 endpoints configured outside the {@code storage.s3.*}
+ * block (e.g. per-source policy config), with the setting named in messages supplied by the
+ * caller.
+ */
+ public static void validateEndpointHost(
+ URI endpoint, boolean allowPrivate, String settingName, String optInHint) {
if (allowPrivate) {
return;
}
String host = endpoint.getHost();
if (host == null || host.isBlank()) {
- throw new IllegalStateException("storage.s3.endpoint must include a host: " + endpoint);
+ throw new IllegalStateException(settingName + " must include a host: " + endpoint);
}
InetAddress[] addresses;
try {
addresses = InetAddress.getAllByName(host);
} catch (UnknownHostException e) {
throw new IllegalStateException(
- "Unable to resolve storage.s3.endpoint host '" + host + "'", e);
+ "Unable to resolve " + settingName + " host '" + host + "'", e);
}
for (InetAddress address : addresses) {
if (isPrivateOrLocal(address)) {
throw new IllegalStateException(
- "storage.s3.endpoint host '"
+ settingName
+ + " host '"
+ host
+ "' resolves to private/link-local address "
+ address.getHostAddress()
- + "; set storage.s3.allow-private-endpoints=true to opt in"
- + " (e.g. for MinIO or in-cluster S3).");
+ + "; "
+ + optInHint);
}
}
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/FleetUsageController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/FleetUsageController.java
index 57b74362ec..8f95920e5a 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/FleetUsageController.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/FleetUsageController.java
@@ -4,6 +4,7 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
+import org.springframework.context.annotation.Profile;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -38,6 +39,9 @@ import stirling.software.proprietary.security.database.repository.UserRepository
@PreAuthorize("hasRole('ADMIN')")
@RequiredArgsConstructor
@EnterpriseEndpoint
+// Self-hosted only: counts are server-wide. On SaaS this endpoint is owned by the team-scoped
+// SaasFleetUsageController (@Profile("saas")) so one backend can't leak another tenant's usage.
+@Profile("!saas")
public class FleetUsageController {
private final PersistentAuditEventRepository auditRepository;
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/LenientEncryptedStringConverter.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/LenientEncryptedStringConverter.java
new file mode 100644
index 0000000000..7bb6190220
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/LenientEncryptedStringConverter.java
@@ -0,0 +1,31 @@
+package stirling.software.proprietary.integration.crypto;
+
+import jakarta.persistence.AttributeConverter;
+import jakarta.persistence.Converter;
+
+/**
+ * {@link EncryptedStringConverter} for columns that held plaintext before encryption shipped:
+ * writes are always encrypted, but a stored value that is not valid ciphertext is returned as-is,
+ * so pre-encryption rows keep loading and become encrypted on their next save. The discrimination
+ * is exact for JSON payloads, which can never be mistaken for ciphertext ('{' is not in the Base64
+ * alphabet). The trade-off is that a genuinely corrupted ciphertext surfaces as garbage to the
+ * caller's parser instead of failing here.
+ */
+@Converter
+public class LenientEncryptedStringConverter implements AttributeConverter {
+
+ @Override
+ public String convertToDatabaseColumn(String attribute) {
+ return CredentialEncryption.encrypt(attribute);
+ }
+
+ @Override
+ public String convertToEntityAttribute(String dbData) {
+ try {
+ return CredentialEncryption.decrypt(dbData);
+ } catch (IllegalArgumentException | IllegalStateException e) {
+ // Not ciphertext: legacy plaintext from before encryption shipped.
+ return dbData;
+ }
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/audit/InfraAuditSummary.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/audit/InfraAuditSummary.java
index c924218e3a..835344e81d 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/audit/InfraAuditSummary.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/audit/InfraAuditSummary.java
@@ -17,6 +17,9 @@ public class InfraAuditSummary {
@Schema(description = "Total events in the returned window", example = "40")
private int totalEvents;
+ @Schema(description = "Policy-run events", example = "3")
+ private int policy;
+
@Schema(description = "Processing-category events", example = "24")
private int processing;
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java
index 9494317751..e2c89f6a54 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java
@@ -23,6 +23,8 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.context.request.RequestContextHolder;
+import org.springframework.web.context.request.ServletRequestAttributes;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
@@ -32,6 +34,7 @@ import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
@@ -42,14 +45,18 @@ import stirling.software.common.model.job.JobResponse;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
+import stirling.software.proprietary.audit.AuditContext;
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.PolicyValidator;
+import stirling.software.proprietary.policy.engine.SweepOutcome;
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
+import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineDefinition;
+import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
@@ -58,12 +65,15 @@ import stirling.software.proprietary.policy.model.PolicyRunView;
import stirling.software.proprietary.policy.overview.PoliciesOverviewResponse;
import stirling.software.proprietary.policy.overview.PolicyOverviewService;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
+import stirling.software.proprietary.policy.source.EditorSource;
import stirling.software.proprietary.policy.source.SourceAccessGuard;
+import stirling.software.proprietary.policy.source.SourceDocCounter;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
import stirling.software.proprietary.policy.trigger.TriggerInfo;
+import stirling.software.proprietary.util.SecretMasker;
/**
* Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
@@ -83,6 +93,7 @@ public class PolicyController {
private final PolicyStore policyStore;
private final SourceStore sourceStore;
private final SourceAccessGuard sourceAccessGuard;
+ private final SourceDocCounter docCounter;
private final PolicyValidator policyValidator;
private final PolicyAccessGuard policyAccessGuard;
private final PolicyManagementAuthority policyManagementAuthority;
@@ -107,11 +118,13 @@ public class PolicyController {
@RequestPart("json") PipelineDefinition definition,
@Valid @ModelAttribute PolicyRunFiles files)
throws IOException {
+ stampPolicyAudit(definition);
requireRunnable(definition);
PolicyInputs inputs = toInputs(files);
- String runId =
- policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId();
- return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
+ PolicyRunHandle handle =
+ policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP);
+ recordEditorDocs(inputs);
+ return ResponseEntity.accepted().body(new JobResponse<>(true, handle.runId(), null));
}
@PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@@ -125,6 +138,7 @@ public class PolicyController {
@RequestPart("json") PipelineDefinition definition,
@Valid @ModelAttribute PolicyRunFiles files)
throws IOException {
+ stampPolicyAudit(definition);
requireRunnable(definition);
PolicyInputs inputs = toInputs(files);
@@ -133,6 +147,7 @@ public class PolicyController {
emitter.onError(e -> log.warn("Policy run SSE emitter error", e));
PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter));
+ recordEditorDocs(inputs);
// whenComplete runs on the worker thread after the run finishes, so the terminal event
// never races the step events.
handle.completion()
@@ -202,7 +217,7 @@ public class PolicyController {
+ " assigned; returns the stored policy with its id.")
public ResponseEntity savePolicy(@RequestBody Policy policy) {
requirePolicyEditingAllowed();
- Policy owned = resolveOwnership(policy);
+ Policy owned = withStoredOutputSecrets(resolveOwnership(policy));
requireAccessibleSources(owned);
try {
policyValidator.validate(owned);
@@ -213,7 +228,7 @@ public class PolicyController {
// Re-sync trigger registrations now so a new/changed folder-watch policy starts being
// watched immediately instead of after the next reconcile sweep.
policyTriggerManager.notifyPoliciesChanged();
- return ResponseEntity.ok(saved);
+ return ResponseEntity.ok(withMaskedOutputSecrets(saved));
}
@PutMapping("/order")
@@ -282,6 +297,50 @@ public class PolicyController {
teamId);
}
+ /** Output secrets never leave the server: reads return the redaction sentinel instead. */
+ private static Policy withMaskedOutputSecrets(Policy policy) {
+ return withOutput(
+ policy,
+ new OutputSpec(
+ policy.output().type(), SecretMasker.mask(policy.output().options())));
+ }
+
+ /**
+ * An edit that round-trips a masked read sends output secrets back as the sentinel; restore
+ * them from the stored policy so saving without re-typing keeps them (validation then runs
+ * against the real values).
+ */
+ private Policy withStoredOutputSecrets(Policy incoming) {
+ if (incoming.id() == null || incoming.id().isBlank()) {
+ return incoming;
+ }
+ return policyStore
+ .get(incoming.id())
+ .map(
+ existing ->
+ withOutput(
+ incoming,
+ new OutputSpec(
+ incoming.output().type(),
+ SecretMasker.restoreRedacted(
+ incoming.output().options(),
+ existing.output().options()))))
+ .orElse(incoming);
+ }
+
+ private static Policy withOutput(Policy policy, OutputSpec output) {
+ return new Policy(
+ policy.id(),
+ policy.name(),
+ policy.owner(),
+ policy.enabled(),
+ policy.trigger(),
+ policy.sourceIds(),
+ policy.steps(),
+ output,
+ policy.teamId());
+ }
+
/**
* Creating, editing, pausing/resuming, and deleting policies requires the editor role for the
* caller's team — a team leader on SaaS (see {@link PolicyManagementAuthority}); the global
@@ -306,9 +365,14 @@ public class PolicyController {
@GetMapping
@Operation(
summary = "List policies",
- description = "Lists the policies belonging to the caller's team.")
+ description =
+ "Lists the policies belonging to the caller's team. Secret-bearing output"
+ + " options are returned as a redaction sentinel, never their stored"
+ + " values.")
public List listPolicies() {
- return policyAccessGuard.visibleFrom(policyStore);
+ return policyAccessGuard.visibleFrom(policyStore).stream()
+ .map(PolicyController::withMaskedOutputSecrets)
+ .toList();
}
@GetMapping("/overview")
@@ -337,11 +401,17 @@ public class PolicyController {
}
@GetMapping("/{policyId}")
- @Operation(summary = "Get a policy by id")
+ @Operation(
+ summary = "Get a policy by id",
+ description =
+ "Secret-bearing output options are returned as a redaction sentinel, never"
+ + " their stored values; an edit that sends the sentinel back keeps"
+ + " them.")
public ResponseEntity getPolicy(@PathVariable String policyId) {
return policyStore
.get(policyId)
.filter(policyAccessGuard::canAccess)
+ .map(PolicyController::withMaskedOutputSecrets)
.map(ResponseEntity::ok)
.orElseGet(() -> ResponseEntity.notFound().build());
}
@@ -401,6 +471,7 @@ public class PolicyController {
() ->
new ResponseStatusException(
HttpStatus.NOT_FOUND, "No policy: " + policyId));
+ stampPolicyAudit(policy.toDefinition());
PolicyInputs inputs = toInputs(files);
String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
@@ -412,9 +483,10 @@ public class PolicyController {
description =
"Pulls the policy's configured sources and runs the pipeline now, regardless of"
+ " the enabled flag (which only gates automatic triggering). Returns"
- + " the ids of the runs started; poll the run-status endpoint for each."
- + " Empty when the sources yielded no work to do.")
- public ResponseEntity> trigger(@PathVariable String policyId) {
+ + " the ids of the runs started (poll the run-status endpoint for each)"
+ + " plus what the sweep skipped - already-processed, parked-by-failure,"
+ + " and in-flight counts - so an empty result explains itself.")
+ public ResponseEntity trigger(@PathVariable String policyId) {
Policy policy =
policyStore
.get(policyId)
@@ -433,6 +505,42 @@ public class PolicyController {
}
}
+ /**
+ * Stamp this run's policy name and step endpoints onto the current request so the audit aspect
+ * can label the event as the policy it ran (rather than the generic {@code /run} endpoint) and
+ * record which tools it executed. No-op outside a web request.
+ */
+ private static void stampPolicyAudit(PipelineDefinition definition) {
+ if (definition == null
+ || !(RequestContextHolder.getRequestAttributes()
+ instanceof ServletRequestAttributes attrs)) {
+ return;
+ }
+ HttpServletRequest request = attrs.getRequest();
+ if (definition.name() != null && !definition.name().isBlank()) {
+ request.setAttribute(AuditContext.REQ_ATTR_POLICY_NAME, definition.name());
+ }
+ List steps =
+ definition.steps().stream()
+ .map(PipelineStep::operation)
+ .filter(op -> op != null && !op.isBlank())
+ .toList();
+ if (!steps.isEmpty()) {
+ request.setAttribute(AuditContext.REQ_ATTR_POLICY_STEPS, steps);
+ }
+ }
+
+ /**
+ * Ad-hoc runs (AI / one-off pipelines) are still editor activity, so their supplied documents
+ * feed the same virtual editor source as stored editor policies, counted against the caller's
+ * team. A run with no primary documents (generator pipeline) records nothing.
+ */
+ private void recordEditorDocs(PolicyInputs inputs) {
+ docCounter.record(
+ EditorSource.counterKey(sourceAccessGuard.currentTeamId()),
+ inputs.primary().size());
+ }
+
/**
* Turn the typed run files into engine {@link PolicyInputs}: the primary documents plus the
* named supporting-file store, where each asset's {@code key} is the name a step references
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java
index 98a6f90096..d54caedd14 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java
@@ -21,7 +21,9 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
+import stirling.software.common.service.AutomationRunContext;
import stirling.software.common.service.FileStorage;
+import stirling.software.common.service.InternalApiClient;
import stirling.software.common.service.InternalApiTimeoutException;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.service.JobQueue;
@@ -148,6 +150,7 @@ public class PolicyEngine {
runAsPrincipal(
billingPrincipal,
fileOwner,
+ definition.name(),
() -> runToCompletion(run, inputs, tracking, completion));
// One admission unit per run; steps run synchronously within it, so this gates heavy work
@@ -199,63 +202,78 @@ public class PolicyEngine {
PolicyProgressListener listener,
CompletableFuture completion) {
String runId = run.getRunId();
- try {
- run.markRunning();
- PolicyExecutionResult result =
- stepExecutor.execute(run.getDefinition(), inputs, listener);
- OutputSpec output = run.getDefinition().output();
- List outputs =
- sinkFor(output)
- .deliver(
- new OutputDelivery(runId, run.getPolicyId()),
- result.files(),
- output);
- taskManager.setMultipleFileResults(runId, outputs);
- taskManager.setComplete(runId);
- run.complete(outputs);
- } catch (PolicyInputRequiredException e) {
- // Expected path: suspend rather than fail. Persist intermediates as fileIds so the run
- // can resume after this worker thread is gone.
- WaitState wait = suspend(e);
- run.waitForInput(wait);
- taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
- } catch (InternalApiTimeoutException e) {
- String message = toolTimeoutMessage(e);
- log.error(
- "Policy run {} timed out on {}: {}",
- runId,
- e.getEndpointPath(),
- e.getMessage());
- run.fail(message);
- taskManager.setError(runId, message);
- } catch (RestClientResponseException e) {
- // A downstream tool call returned an error status. When it's a structured entitlement
- // response (401/402 with a JSON `error` sentinel), surface that code onto the run so
- // the
- // client can react — e.g. pop the usage-limit modal — instead of only seeing a generic
- // failure. We don't interpret the code here (that would couple this module to the saas
- // billing layer); we just pass it through for the client to map. Other statuses fall
- // through to the generic failure below.
- String code = DownstreamEntitlementError.extractCode(e);
- if (code != null) {
- log.info("Policy run {} blocked by downstream entitlement gate ({})", runId, code);
- String message = "Usage limit reached";
- run.failWithCode(message, code, DownstreamEntitlementError.extractSubscribed(e));
- taskManager.setError(runId, message);
- } else {
- String message = "Policy run failed: " + e.getMessage();
- log.error("Policy run {} failed (downstream HTTP error)", runId, e);
+ // One policy run = one automation run. Scope the run id on this worker thread (the async
+ // hop already happened) so every tool sub-step dispatched via InternalApiClient groups into
+ // a single charge, and two separate policy runs on the same document stay distinct charges.
+ try (AutomationRunContext.Scope runScope = AutomationRunContext.open(runId)) {
+ try {
+ run.markRunning();
+ PolicyExecutionResult result =
+ stepExecutor.execute(run.getDefinition(), inputs, listener);
+ OutputSpec output = run.getDefinition().output();
+ List outputs =
+ sinkFor(output)
+ .deliver(
+ new OutputDelivery(runId, run.getPolicyId()),
+ result.files(),
+ output);
+ taskManager.setMultipleFileResults(runId, outputs);
+ taskManager.setComplete(runId);
+ run.complete(outputs);
+ } catch (PolicyInputRequiredException e) {
+ // Expected path: suspend rather than fail. Persist intermediates as fileIds so the
+ // run
+ // can resume after this worker thread is gone.
+ WaitState wait = suspend(e);
+ run.waitForInput(wait);
+ taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
+ } catch (InternalApiTimeoutException e) {
+ String message = toolTimeoutMessage(e);
+ log.error(
+ "Policy run {} timed out on {}: {}",
+ runId,
+ e.getEndpointPath(),
+ e.getMessage());
run.fail(message);
taskManager.setError(runId, message);
+ } catch (RestClientResponseException e) {
+ // A downstream tool call returned an error status. When it's a structured
+ // entitlement
+ // response (401/402 with a JSON `error` sentinel), surface that code onto the run
+ // so
+ // the
+ // client can react — e.g. pop the usage-limit modal — instead of only seeing a
+ // generic
+ // failure. We don't interpret the code here (that would couple this module to the
+ // saas
+ // billing layer); we just pass it through for the client to map. Other statuses
+ // fall
+ // through to the generic failure below.
+ String code = DownstreamEntitlementError.extractCode(e);
+ if (code != null) {
+ log.info(
+ "Policy run {} blocked by downstream entitlement gate ({})",
+ runId,
+ code);
+ String message = "Usage limit reached";
+ run.failWithCode(
+ message, code, DownstreamEntitlementError.extractSubscribed(e));
+ taskManager.setError(runId, message);
+ } else {
+ String message = "Policy run failed: " + e.getMessage();
+ log.error("Policy run {} failed (downstream HTTP error)", runId, e);
+ run.fail(message);
+ taskManager.setError(runId, message);
+ }
+ } catch (Exception e) {
+ String message = "Policy run failed: " + e.getMessage();
+ log.error("Policy run {} failed", runId, e);
+ run.fail(message);
+ taskManager.setError(runId, message);
+ } finally {
+ // Always resolve so stream/await callers unblock.
+ completion.complete(run);
}
- } catch (Exception e) {
- String message = "Policy run failed: " + e.getMessage();
- log.error("Policy run {} failed", runId, e);
- run.fail(message);
- taskManager.setError(runId, message);
- } finally {
- // Always resolve so stream/await callers unblock.
- completion.complete(run);
}
}
@@ -361,18 +379,26 @@ public class PolicyEngine {
* dispatch attributes (and charges) usage to that user. A null/blank principal runs as-is.
* Restores the previous MDC value afterward (defensive — worker threads aren't pooled).
*/
- private static void runAsPrincipal(String billingPrincipal, String fileOwner, Runnable body) {
+ private static void runAsPrincipal(
+ String billingPrincipal, String fileOwner, String policyName, Runnable body) {
// Billing identity (MDC auditPrincipal) and output-file ownership (JobContext owner) are
// set
// independently: usage is charged to billingPrincipal, but stored output files are owned by
// fileOwner — the user who triggered an org-wide policy — so they can fetch their results.
// Either may be null (e.g. login disabled, or a trigger-fired run); each is applied only
- // when present and restored afterward (defensive — worker threads aren't pooled).
+ // when present and restored afterward (defensive — worker threads aren't pooled). The
+ // policy
+ // name rides MDC too so each tool step's loopback dispatch (InternalApiClient) can forward
+ // it as a header, letting the audit tie the step back to its policy.
String previousPrincipal = MDC.get(AUDIT_PRINCIPAL_MDC_KEY);
+ String previousPolicyName = MDC.get(InternalApiClient.POLICY_NAME_MDC_KEY);
String previousOwner = JobContext.getOwner();
if (billingPrincipal != null && !billingPrincipal.isBlank()) {
MDC.put(AUDIT_PRINCIPAL_MDC_KEY, billingPrincipal);
}
+ if (policyName != null && !policyName.isBlank()) {
+ MDC.put(InternalApiClient.POLICY_NAME_MDC_KEY, policyName);
+ }
if (fileOwner != null && !fileOwner.isBlank()) {
JobContext.setOwner(fileOwner);
}
@@ -384,6 +410,11 @@ public class PolicyEngine {
} else {
MDC.remove(AUDIT_PRINCIPAL_MDC_KEY);
}
+ if (previousPolicyName != null) {
+ MDC.put(InternalApiClient.POLICY_NAME_MDC_KEY, previousPolicyName);
+ } else {
+ MDC.remove(InternalApiClient.POLICY_NAME_MDC_KEY);
+ }
JobContext.setOwner(previousOwner);
}
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java
index 5a35b879c0..8575e95e61 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java
@@ -21,6 +21,7 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
+import stirling.software.proprietary.policy.source.EditorSource;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceDocCounter;
import stirling.software.proprietary.policy.source.SourceStore;
@@ -44,7 +45,7 @@ public class PolicyRunner {
private final ProcessedLedger processedLedger;
/** Full-listing sweep: resolve every source, then reconcile the ledger. */
- public List run(Policy policy) {
+ public SweepOutcome run(Policy policy) {
return run(policy, SweepKind.FULL);
}
@@ -52,10 +53,10 @@ public class PolicyRunner {
* Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
* one failure does not affect the others. No sources means one run with no input (generator
* pipeline). Missing or disabled sources are skipped so one broken reference does not stop the
- * rest. Returns the ids of the runs it started (empty when sources yielded no work), so a
- * manual trigger can report back which runs to follow.
+ * rest. Returns the ids of the runs it started plus what the sweep skipped, so a manual trigger
+ * can report which runs to follow or why nothing ran.
*/
- public List run(Policy policy, SweepKind sweep) {
+ public SweepOutcome run(Policy policy, SweepKind sweep) {
long sweepStart = System.currentTimeMillis();
PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger);
List runIds = new ArrayList<>();
@@ -95,13 +96,19 @@ public class PolicyRunner {
policy.id());
}
}
- return runIds;
+ return context.outcome(runIds);
}
- /** Run a stored policy on caller-supplied files (e.g. manual upload), bypassing its sources. */
+ /**
+ * Run a stored policy on caller-supplied files (e.g. an editor upload), bypassing its sources.
+ * The supplied documents are still counted against the virtual {@link EditorSource}, scoped to
+ * the policy's team, so the Sources overview reports the whole team's editor throughput.
+ */
public PolicyRunHandle runWith(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
- return policyEngine.runPolicy(policy, inputs, listener);
+ PolicyRunHandle handle = policyEngine.runPolicy(policy, inputs, listener);
+ docCounter.record(EditorSource.counterKey(policy.teamId()), inputs.primary().size());
+ return handle;
}
/** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java
index 5302bacdb3..469b5f769e 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java
@@ -86,4 +86,32 @@ final class PolicySweep implements ResolveContext {
synchronized Set presentIdentities() {
return Set.copyOf(present);
}
+
+ /**
+ * Summarise the sweep from state already in hand (no extra ledger reads): the prefetched rows
+ * were loaded before claiming, and successful claims flipped their entries to PROCESSING, so
+ * what remains DONE or ERROR is exactly what this sweep skipped.
+ */
+ synchronized SweepOutcome outcome(List runIds) {
+ int alreadyProcessed = 0;
+ int parked = 0;
+ int processing = 0;
+ for (String identity : present) {
+ ClaimState state = prefetched.get(identity);
+ if (state == null) {
+ continue;
+ }
+ switch (state.status()) {
+ case DONE -> alreadyProcessed++;
+ case ERROR -> parked++;
+ case PROCESSING, INTERRUPTED -> processing++;
+ }
+ }
+ return new SweepOutcome(
+ runIds,
+ present.size(),
+ alreadyProcessed,
+ parked,
+ Math.max(0, processing - runIds.size()));
+ }
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/SweepOutcome.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/SweepOutcome.java
new file mode 100644
index 0000000000..752299976d
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/SweepOutcome.java
@@ -0,0 +1,19 @@
+package stirling.software.proprietary.policy.engine;
+
+import java.util.List;
+
+/**
+ * What one policy sweep found and started, so a manual trigger can explain an empty result instead
+ * of a blanket "nothing to do": how many files the sources listed, how many were skipped because
+ * they are already processed at their current version, how many are parked by a failed run (not
+ * retried until they change or history is cleared), and how many are still in flight from an
+ * earlier sweep. Counts are zero for {@link SweepKind#LIGHT} sweeps, which do not take a full
+ * listing.
+ */
+public record SweepOutcome(
+ List runIds, int filesListed, int alreadyProcessed, int parked, int inFlight) {
+
+ public SweepOutcome {
+ runIds = runIds == null ? List.of() : List.copyOf(runIds);
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/S3InputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/S3InputSource.java
new file mode 100644
index 0000000000..99e189f326
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/S3InputSource.java
@@ -0,0 +1,286 @@
+package stirling.software.proprietary.policy.input;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
+import org.springframework.core.io.AbstractResource;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.proprietary.policy.model.InputSpec;
+import stirling.software.proprietary.policy.model.PolicyInputs;
+import stirling.software.proprietary.policy.s3.S3Config;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+import stirling.software.proprietary.policy.s3.S3Identities;
+
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+/**
+ * Reads input files from an Amazon S3 (or S3-compatible) bucket; each listed object is its own unit
+ * of work, claimed through the {@link ResolveContext} ledger and tracked in place. Identity and
+ * version gate come from {@link S3Identities}, so the steady-state sweep never downloads content.
+ * Options (see {@link S3Config}): "bucket" (required), "region" (default us-east-1), "prefix" (only
+ * keys starting with it are read), "endpoint" (S3-compatible stores such as MinIO; path-style
+ * addressing is used automatically), "accessKeyId" and "secretAccessKey" (required; requests are
+ * never signed with the server's own AWS identity), and "mode" which is "consume" (default: a
+ * processed object is deleted once every policy that claimed it has settled successfully and it is
+ * still the version that ran; failures stay in place and are not retried until they change) or
+ * "snapshot" (stateless, every run sees the full set). Keys ending in "/" (folder placeholders) and
+ * keys with a dot-prefixed path segment are never picked up, mirroring the folder source's
+ * hidden-file rule.
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@ConditionalOnBooleanProperty(name = "policies.enabled")
+public class S3InputSource implements InputSource {
+
+ private static final String TYPE = "s3";
+
+ private final S3ConnectionPool connectionPool;
+
+ @Override
+ public String type() {
+ return TYPE;
+ }
+
+ @Override
+ public boolean supports(InputSpec spec) {
+ return spec != null && TYPE.equals(spec.type());
+ }
+
+ /**
+ * Fails fast at save time: bad config shape, a private endpoint without the operator opt-in, or
+ * a bucket the supplied credentials cannot list.
+ */
+ @Override
+ public void validate(InputSpec spec) {
+ S3Config config = S3Config.from(spec.options());
+ try {
+ connectionPool.clientFor(config).listObjectsV2(listRequest(config).maxKeys(1).build());
+ } catch (SdkException e) {
+ throw new IllegalArgumentException(
+ "cannot access s3://"
+ + config.bucket()
+ + "/"
+ + config.prefix()
+ + ": "
+ + e.getMessage(),
+ e);
+ }
+ }
+
+ @Override
+ public List resolve(InputSpec spec, ResolveContext ctx) throws IOException {
+ S3Config config = S3Config.from(spec.options());
+ S3Client client = connectionPool.clientFor(config);
+ // A listing failure propagates so the sweep reads it as "could not list" (which vetoes
+ // presence cleanup), never as "verifiably no objects".
+ List objects = listObjects(client, config);
+
+ if (config.snapshot()) {
+ return objects.stream()
+ .map(
+ object ->
+ ResolvedInput.of(
+ PolicyInputs.of(
+ List.of(
+ objectResource(
+ client, config, object)))))
+ .toList();
+ }
+
+ ctx.reportPresent(
+ objects.stream()
+ .map(object -> S3Identities.identity(config.bucket(), object.key()))
+ .toList());
+
+ List work = new ArrayList<>();
+ for (S3Object object : objects) {
+ String identity = S3Identities.identity(config.bucket(), object.key());
+ String gate = S3Identities.gate(object.eTag(), object.size(), object.lastModified());
+ if (!ctx.claim(identity, gate, null)) {
+ continue;
+ }
+ work.add(
+ new ResolvedInput(
+ PolicyInputs.of(List.of(objectResource(client, config, object))),
+ success ->
+ completeConsumed(
+ ctx,
+ client,
+ config,
+ object.key(),
+ identity,
+ gate,
+ success)));
+ }
+ return work;
+ }
+
+ /**
+ * Settle at the version this run claimed, then remove the object only when it still carries
+ * that version and every policy that claimed it has settled DONE, mirroring the folder source's
+ * consensus delete. A failed run settles ERROR and never deletes; the DONE row of an object
+ * that could not be deleted still stops reprocessing.
+ */
+ private void completeConsumed(
+ ResolveContext ctx,
+ S3Client client,
+ S3Config config,
+ String key,
+ String identity,
+ String claimGate,
+ boolean success) {
+ ctx.settle(identity, claimGate, null, success);
+ if (!success) {
+ return;
+ }
+ try {
+ HeadObjectResponse head =
+ client.headObject(
+ HeadObjectRequest.builder().bucket(config.bucket()).key(key).build());
+ String currentGate =
+ S3Identities.gate(head.eTag(), head.contentLength(), head.lastModified());
+ if (currentGate.equals(claimGate) && ctx.allSettledDone(identity)) {
+ client.deleteObject(
+ DeleteObjectRequest.builder().bucket(config.bucket()).key(key).build());
+ }
+ } catch (NoSuchKeyException alreadyGone) {
+ // Removed by the user or a co-watching policy's own consensus delete: nothing to do.
+ } catch (S3Exception e) {
+ if (e.statusCode() == 404) {
+ return;
+ }
+ log.warn("Could not remove consumed S3 object {}: {}", identity, e.getMessage());
+ } catch (SdkException e) {
+ log.warn("Could not remove consumed S3 object {}: {}", identity, e.getMessage());
+ }
+ }
+
+ /** Every ingestible object under the configured prefix, across all listing pages. */
+ private static List listObjects(S3Client client, S3Config config) {
+ List objects = new ArrayList<>();
+ String continuationToken = null;
+ do {
+ ListObjectsV2Request.Builder request = listRequest(config);
+ if (continuationToken != null) {
+ request.continuationToken(continuationToken);
+ }
+ ListObjectsV2Response page = client.listObjectsV2(request.build());
+ for (S3Object object : page.contents()) {
+ if (ingestible(object)) {
+ objects.add(object);
+ }
+ }
+ continuationToken = page.nextContinuationToken();
+ } while (continuationToken != null);
+ return objects;
+ }
+
+ private static ListObjectsV2Request.Builder listRequest(S3Config config) {
+ ListObjectsV2Request.Builder request =
+ ListObjectsV2Request.builder().bucket(config.bucket());
+ if (!config.prefix().isEmpty()) {
+ request.prefix(config.prefix());
+ }
+ return request;
+ }
+
+ /**
+ * Folder-placeholder keys (ending "/") and keys with a dot-prefixed segment are skipped, so a
+ * hidden convention (e.g. a future output sink's staging prefix) is never re-ingested.
+ */
+ private static boolean ingestible(S3Object object) {
+ String key = object.key();
+ if (key.isEmpty() || key.endsWith("/")) {
+ return false;
+ }
+ for (String segment : key.split("/")) {
+ if (segment.startsWith(".")) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static Resource objectResource(S3Client client, S3Config config, S3Object object) {
+ return new S3ObjectResource(client, config.bucket(), object);
+ }
+
+ /**
+ * Streams the object on demand, pinned to the ETag observed at listing time so a run never
+ * reads a different version than the sweep claimed (a swapped object fails the read with a
+ * precondition error and the new version is claimed by a later sweep).
+ */
+ private static final class S3ObjectResource extends AbstractResource {
+
+ private final S3Client client;
+ private final String bucket;
+ private final String key;
+ private final String eTag;
+ private final Long size;
+
+ private S3ObjectResource(S3Client client, String bucket, S3Object object) {
+ this.client = client;
+ this.bucket = bucket;
+ this.key = object.key();
+ this.eTag = object.eTag();
+ this.size = object.size();
+ }
+
+ @Override
+ public InputStream getInputStream() throws IOException {
+ GetObjectRequest.Builder request = GetObjectRequest.builder().bucket(bucket).key(key);
+ if (eTag != null && !eTag.isBlank()) {
+ request.ifMatch(eTag);
+ }
+ try {
+ return client.getObject(request.build());
+ } catch (NoSuchKeyException e) {
+ throw new FileNotFoundException(getDescription() + " no longer exists");
+ } catch (SdkException e) {
+ throw new IOException(
+ "Could not read " + getDescription() + ": " + e.getMessage(), e);
+ }
+ }
+
+ /** Listed just now; readers get a precise error from {@link #getInputStream} instead. */
+ @Override
+ public boolean exists() {
+ return true;
+ }
+
+ @Override
+ public long contentLength() {
+ return size == null ? -1 : size;
+ }
+
+ @Override
+ public String getFilename() {
+ return key.substring(key.lastIndexOf('/') + 1);
+ }
+
+ @Override
+ public String getDescription() {
+ return "S3 object " + S3Identities.identity(bucket, key);
+ }
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java
index ed3cf707ad..52da21208a 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java
@@ -15,7 +15,6 @@ import java.util.List;
import java.util.UUID;
import java.util.stream.Stream;
-import org.apache.commons.io.FilenameUtils;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
@@ -82,7 +81,7 @@ public class FolderOutputSink implements PolicyOutputSink {
List results = new ArrayList<>();
for (int i = 0; i < outputs.size(); i++) {
Resource resource = outputs.get(i);
- String name = safeName(resource.getFilename(), i);
+ String name = OutputNames.safeName(resource.getFilename(), i);
Path staged = tmpDir.resolve(UUID.randomUUID().toString());
String contentHash = stage(resource, staged, delivery.policyId() != null);
long size = Files.size(staged);
@@ -198,29 +197,14 @@ public class FolderOutputSink implements PolicyOutputSink {
return Path.of(directory.toString());
}
- // Strip any directory component / "../" so a crafted output name cannot escape targetDir.
- private static String safeName(String filename, int index) {
- if (filename == null || filename.isBlank()) {
- return "output-" + index;
- }
- String name = FilenameUtils.getName(filename);
- if (name.isBlank() || ".".equals(name) || "..".equals(name)) {
- return "output-" + index;
- }
- return name;
- }
-
// Non-colliding path, appending " (n)" before the extension.
private static Path uniqueTarget(Path dir, String filename) {
Path candidate = dir.resolve(filename);
if (!Files.exists(candidate)) {
return candidate;
}
- String base = FilenameUtils.getBaseName(filename);
- String ext = FilenameUtils.getExtension(filename);
- String suffix = ext.isEmpty() ? "" : "." + ext;
for (int n = 1; ; n++) {
- Path next = dir.resolve(base + " (" + n + ")" + suffix);
+ Path next = dir.resolve(OutputNames.numbered(filename, n));
if (!Files.exists(next)) {
return next;
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/OutputNames.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/OutputNames.java
new file mode 100644
index 0000000000..e82cc66b28
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/OutputNames.java
@@ -0,0 +1,29 @@
+package stirling.software.proprietary.policy.output;
+
+import org.apache.commons.io.FilenameUtils;
+
+/** Output file naming shared by the sinks: sanitised base names and collision suffixes. */
+final class OutputNames {
+
+ private OutputNames() {}
+
+ /** Strip any directory component / "../" so a crafted output name cannot escape the target. */
+ static String safeName(String filename, int index) {
+ if (filename == null || filename.isBlank()) {
+ return "output-" + index;
+ }
+ String name = FilenameUtils.getName(filename);
+ if (name.isBlank() || ".".equals(name) || "..".equals(name)) {
+ return "output-" + index;
+ }
+ return name;
+ }
+
+ /** The nth alternative for a taken name, appending " (n)" before the extension. */
+ static String numbered(String filename, int n) {
+ String base = FilenameUtils.getBaseName(filename);
+ String ext = FilenameUtils.getExtension(filename);
+ String suffix = ext.isEmpty() ? "" : "." + ext;
+ return base + " (" + n + ")" + suffix;
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java
new file mode 100644
index 0000000000..c7d740868a
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java
@@ -0,0 +1,270 @@
+package stirling.software.proprietary.policy.output;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.UUID;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
+import org.springframework.core.io.Resource;
+import org.springframework.http.MediaType;
+import org.springframework.http.MediaTypeFactory;
+import org.springframework.stereotype.Service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.model.job.ResultFile;
+import stirling.software.proprietary.policy.ledger.ProcessedLedger;
+import stirling.software.proprietary.policy.model.OutputSpec;
+import stirling.software.proprietary.policy.s3.S3Config;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+import stirling.software.proprietary.policy.s3.S3Identities;
+
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+/**
+ * Uploads a run's outputs to the bucket and key prefix given in the {@link OutputSpec} (same
+ * connection options as the S3 input source; "prefix" is the destination folder). The
+ * record-before-visible obligation is met without a rename step: a single-part PUT's ETag is the
+ * MD5 of its content on plain and SSE-S3 buckets, so the ledger row is recorded at that predicted
+ * gate BEFORE the upload, and the object is claimed under exactly the gate the next listing
+ * returns. Stores where the returned ETag differs (e.g. SSE-KMS) are re-recorded at the actual gate
+ * immediately after the PUT - a narrow race those buckets accept rather than a broken loop. Names
+ * never overwrite: uploads are conditional on the key not existing ({@code If-None-Match: *}),
+ * re-picking "name (n).ext" on collision exactly like the folder sink; stores without
+ * conditional-write support fall back to an existence check per candidate.
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@ConditionalOnBooleanProperty(name = "policies.enabled")
+public class S3OutputSink implements PolicyOutputSink {
+
+ private static final String TYPE = "s3";
+
+ private final S3ConnectionPool connectionPool;
+ private final ProcessedLedger processedLedger;
+
+ @Override
+ public String type() {
+ return TYPE;
+ }
+
+ @Override
+ public boolean supports(OutputSpec spec) {
+ return spec != null && TYPE.equals(spec.type());
+ }
+
+ /**
+ * Config shape and endpoint guard only - no network probe, since write-only credentials
+ * (s3:PutObject without s3:ListBucket) are a legitimate setup for an output bucket and a
+ * listing probe would wrongly reject them.
+ */
+ @Override
+ public void validate(OutputSpec spec) {
+ connectionPool.clientFor(S3Config.from(spec.options()));
+ }
+
+ @Override
+ public List deliver(
+ OutputDelivery delivery, List outputs, OutputSpec spec) throws IOException {
+ S3Config config = S3Config.from(spec.options());
+ S3Client client = connectionPool.clientFor(config);
+
+ List results = new ArrayList<>();
+ for (int i = 0; i < outputs.size(); i++) {
+ Resource resource = outputs.get(i);
+ String name = OutputNames.safeName(resource.getFilename(), i);
+ Path staged = Files.createTempFile("s3-output-", ".tmp");
+ try {
+ String predictedGate = stage(resource, staged, delivery.policyId() != null);
+ long size = Files.size(staged);
+ String key = upload(delivery, client, config, name, staged, predictedGate);
+ String contentType =
+ MediaTypeFactory.getMediaType(name)
+ .orElse(MediaType.APPLICATION_OCTET_STREAM)
+ .toString();
+ results.add(
+ ResultFile.builder()
+ .fileId(UUID.randomUUID().toString())
+ .fileName(S3Identities.identity(config.bucket(), key))
+ .contentType(contentType)
+ .fileSize(size)
+ .build());
+ log.debug(
+ "Wrote policy run {} output to {}",
+ delivery.runId(),
+ S3Identities.identity(config.bucket(), key));
+ } finally {
+ try {
+ Files.deleteIfExists(staged);
+ } catch (IOException e) {
+ log.warn("Could not remove S3 staging file {}: {}", staged, e.getMessage());
+ }
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Spool the output to a local staging file (S3 needs a known content length, and the body must
+ * be re-readable across collision retries). For a recorded delivery the MD5 - the predicted
+ * single-part ETag - is digested in the same pass; ad-hoc runs record nothing and skip it.
+ */
+ private static String stage(Resource resource, Path staged, boolean recorded)
+ throws IOException {
+ if (!recorded) {
+ try (InputStream is = resource.getInputStream();
+ OutputStream out = Files.newOutputStream(staged)) {
+ is.transferTo(out);
+ }
+ return null;
+ }
+ MessageDigest digest = newMd5();
+ try (InputStream is = resource.getInputStream();
+ DigestOutputStream out =
+ new DigestOutputStream(Files.newOutputStream(staged), digest)) {
+ is.transferTo(out);
+ }
+ return HexFormat.of().formatHex(digest.digest());
+ }
+
+ /**
+ * The S3 shape of the folder sink's record-then-rename loop. The ledger row must exist before
+ * the object is visible, so it is recorded at the predicted gate before the PUT; losing the
+ * chosen key to a concurrent writer (the conditional PUT fails) forgets the just-recorded row -
+ * whatever object actually owns that key must stay claimable at any version - then re-picks. A
+ * PUT that never made the object visible also forgets its row.
+ */
+ private String upload(
+ OutputDelivery delivery,
+ S3Client client,
+ S3Config config,
+ String name,
+ Path staged,
+ String predictedGate)
+ throws IOException {
+ String keyPrefix = keyPrefix(config);
+ boolean conditionalPuts = true;
+ for (int attempt = 0; ; attempt++) {
+ String key = keyPrefix + (attempt == 0 ? name : OutputNames.numbered(name, attempt));
+ String identity = S3Identities.identity(config.bucket(), key);
+ if (!conditionalPuts && exists(client, config.bucket(), key)) {
+ continue;
+ }
+ if (delivery.policyId() != null) {
+ processedLedger.recordOutput(delivery.policyId(), identity, predictedGate, null);
+ }
+ PutObjectRequest.Builder put =
+ PutObjectRequest.builder().bucket(config.bucket()).key(key);
+ if (conditionalPuts) {
+ put.ifNoneMatch("*");
+ }
+ try {
+ PutObjectResponse response =
+ client.putObject(put.build(), RequestBody.fromFile(staged));
+ reRecordIfGateDiffers(delivery, identity, predictedGate, response);
+ return key;
+ } catch (S3Exception e) {
+ forgetRecorded(delivery, identity, predictedGate);
+ if (conditionalPuts && e.statusCode() == 412) {
+ // Known edge: if our own PUT succeeded server-side but the response was lost
+ // and the SDK retried, that retry 412s here too - we then upload under the
+ // next name, leaving the first object row-less (claimable, single duplicate).
+ // Requires a response-lost network flake at exactly this moment; accepted.
+ log.debug("Output key {} taken concurrently; re-picking", identity);
+ continue;
+ }
+ if (conditionalPuts && e.statusCode() == 501) {
+ // Store without conditional-write support: retry this candidate with a plain
+ // existence check instead.
+ log.debug(
+ "Conditional PUT unsupported by {}; falling back to existence checks",
+ config.bucket());
+ conditionalPuts = false;
+ attempt--;
+ continue;
+ }
+ throw new IOException("Could not upload " + identity + ": " + e.getMessage(), e);
+ } catch (SdkException e) {
+ forgetRecorded(delivery, identity, predictedGate);
+ throw new IOException("Could not upload " + identity + ": " + e.getMessage(), e);
+ }
+ }
+ }
+
+ /**
+ * On buckets where a PUT's ETag is not the content MD5 (e.g. SSE-KMS), re-record at the gate
+ * listings will actually return. The row is briefly at the wrong gate while the object is
+ * already visible - the narrow race such stores trade for a working self-output skip.
+ */
+ private void reRecordIfGateDiffers(
+ OutputDelivery delivery,
+ String identity,
+ String predictedGate,
+ PutObjectResponse response) {
+ if (delivery.policyId() == null) {
+ return;
+ }
+ String actualGate = S3Identities.gate(response.eTag(), null, null);
+ if (!actualGate.equals(predictedGate)) {
+ log.debug(
+ "PUT ETag for {} differs from content MD5 (encrypted bucket?); re-recording",
+ identity);
+ processedLedger.recordOutput(delivery.policyId(), identity, actualGate, null);
+ }
+ }
+
+ private void forgetRecorded(OutputDelivery delivery, String identity, String predictedGate) {
+ if (delivery.policyId() != null) {
+ processedLedger.forgetOutput(delivery.policyId(), identity, predictedGate);
+ }
+ }
+
+ private static boolean exists(S3Client client, String bucket, String key) {
+ try {
+ client.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build());
+ return true;
+ } catch (NoSuchKeyException e) {
+ return false;
+ } catch (S3Exception e) {
+ if (e.statusCode() == 404) {
+ return false;
+ }
+ throw e;
+ }
+ }
+
+ /** The configured prefix as a key-path prefix: "processed" and "processed/" mean the same. */
+ private static String keyPrefix(S3Config config) {
+ String prefix = config.prefix();
+ if (prefix.isEmpty() || prefix.endsWith("/")) {
+ return prefix;
+ }
+ return prefix + "/";
+ }
+
+ private static MessageDigest newMd5() {
+ try {
+ return MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("MD5 unavailable", e);
+ }
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java
new file mode 100644
index 0000000000..152a6de4cf
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java
@@ -0,0 +1,103 @@
+package stirling.software.proprietary.policy.s3;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+/**
+ * Connection settings shared by the S3 input source and output sink, parsed from a spec's options
+ * map. Credentials are required: there is deliberately no fallback to the server's own AWS
+ * credential chain, so user-supplied config can never borrow the host's identity. {@code snapshot}
+ * is input-only and ignored by the sink.
+ */
+public record S3Config(
+ String bucket,
+ String region,
+ String prefix,
+ String endpoint,
+ String accessKeyId,
+ String secretAccessKey,
+ boolean snapshot) {
+
+ private static final String BUCKET_OPTION = "bucket";
+ private static final String REGION_OPTION = "region";
+ private static final String PREFIX_OPTION = "prefix";
+ private static final String ENDPOINT_OPTION = "endpoint";
+ private static final String ACCESS_KEY_ID_OPTION = "accessKeyId";
+ private static final String SECRET_ACCESS_KEY_OPTION = "secretAccessKey";
+ private static final String MODE_OPTION = "mode";
+ private static final String MODE_CONSUME = "consume";
+ private static final String MODE_SNAPSHOT = "snapshot";
+
+ public static S3Config from(Map options) {
+ String bucket = trimmed(options.get(BUCKET_OPTION));
+ if (bucket == null) {
+ throw new IllegalArgumentException("s3 config requires a 'bucket' option");
+ }
+ String region = trimmed(options.get(REGION_OPTION));
+ String prefix = trimmed(options.get(PREFIX_OPTION));
+ if (prefix != null && prefix.startsWith("/")) {
+ prefix = prefix.substring(1);
+ }
+ String endpoint = validEndpoint(trimmed(options.get(ENDPOINT_OPTION)));
+ String accessKeyId = trimmed(options.get(ACCESS_KEY_ID_OPTION));
+ String secretAccessKey = trimmed(options.get(SECRET_ACCESS_KEY_OPTION));
+ if (accessKeyId == null || secretAccessKey == null) {
+ throw new IllegalArgumentException(
+ "s3 config requires an 'accessKeyId' and 'secretAccessKey'");
+ }
+ String mode = trimmed(options.get(MODE_OPTION));
+ if (mode != null && !MODE_CONSUME.equals(mode) && !MODE_SNAPSHOT.equals(mode)) {
+ throw new IllegalArgumentException("s3 config 'mode' must be 'consume' or 'snapshot'");
+ }
+ return new S3Config(
+ bucket,
+ region == null ? "us-east-1" : region,
+ prefix == null ? "" : prefix,
+ endpoint,
+ accessKeyId,
+ secretAccessKey,
+ MODE_SNAPSHOT.equals(mode));
+ }
+
+ private static String validEndpoint(String endpoint) {
+ if (endpoint == null) {
+ return null;
+ }
+ URI uri;
+ try {
+ uri = new URI(endpoint);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException("s3 config 'endpoint' is not a valid URL", e);
+ }
+ if (!"http".equals(uri.getScheme()) && !"https".equals(uri.getScheme())) {
+ throw new IllegalArgumentException(
+ "s3 config 'endpoint' must be an http(s) URL, e.g. https://s3.example.com");
+ }
+ return endpoint;
+ }
+
+ private static String trimmed(Object value) {
+ if (value == null) {
+ return null;
+ }
+ String text = value.toString().trim();
+ return text.isEmpty() ? null : text;
+ }
+
+ /** Never prints the credentials, so an accidental log line cannot leak them. */
+ @Override
+ public String toString() {
+ return "S3Config[bucket="
+ + bucket
+ + ", region="
+ + region
+ + ", prefix="
+ + prefix
+ + ", endpoint="
+ + endpoint
+ + ", snapshot="
+ + snapshot
+ + "]";
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3ConnectionPool.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3ConnectionPool.java
new file mode 100644
index 0000000000..4118557dbd
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3ConnectionPool.java
@@ -0,0 +1,110 @@
+package stirling.software.proprietary.policy.s3;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
+import org.springframework.stereotype.Service;
+
+import jakarta.annotation.PreDestroy;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.cluster.s3.S3Clients;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.S3Configuration;
+
+/**
+ * Long-lived {@link S3Client}s for policy S3 sources and sinks, one per distinct {@link S3Config},
+ * closed at shutdown. An edited spec simply maps to a new entry, and a stale entry costs nothing
+ * (the URL-connection HTTP client holds no pooled sockets or threads). Clients sign exclusively
+ * with the spec's own credentials - there is deliberately no fallback to the server's AWS
+ * credential chain, so user-supplied config can never borrow the host's identity. Endpoints are
+ * guarded against private addresses before a client is ever built, since they come from portal
+ * users rather than the operator.
+ */
+@Service
+@ConditionalOnBooleanProperty(name = "policies.enabled")
+public class S3ConnectionPool {
+
+ private final ApplicationProperties applicationProperties;
+ private final Function clientFactory;
+ private final Map clients = new ConcurrentHashMap<>();
+
+ @Autowired
+ public S3ConnectionPool(ApplicationProperties applicationProperties) {
+ this(applicationProperties, S3ConnectionPool::buildClient);
+ }
+
+ /** Factory-injecting constructor for tests. */
+ public S3ConnectionPool(
+ ApplicationProperties applicationProperties,
+ Function clientFactory) {
+ this.applicationProperties = applicationProperties;
+ this.clientFactory = clientFactory;
+ }
+
+ public S3Client clientFor(S3Config config) {
+ return clients.computeIfAbsent(
+ config,
+ c -> {
+ requirePermittedEndpoint(c);
+ return clientFactory.apply(c);
+ });
+ }
+
+ /**
+ * A user-supplied endpoint must not reach loopback, link-local, or private addresses unless the
+ * operator has opted in via {@code policies.allowPrivateS3Endpoints}.
+ */
+ private void requirePermittedEndpoint(S3Config config) {
+ if (config.endpoint() == null) {
+ return;
+ }
+ try {
+ S3Clients.validateEndpointHost(
+ URI.create(config.endpoint()),
+ applicationProperties.getPolicies().isAllowPrivateS3Endpoints(),
+ "S3 source endpoint",
+ "set policies.allowPrivateS3Endpoints=true to opt in (e.g. for a local"
+ + " MinIO).");
+ } catch (IllegalStateException e) {
+ throw new IllegalArgumentException(e.getMessage(), e);
+ }
+ }
+
+ private static S3Client buildClient(S3Config config) {
+ S3ClientBuilder builder =
+ S3Client.builder()
+ .httpClient(UrlConnectionHttpClient.create())
+ .region(Region.of(config.region()))
+ // Path-style addressing whenever a custom endpoint is set: S3-compatible
+ // stores rarely support virtual-hosted bucket DNS.
+ .serviceConfiguration(
+ S3Configuration.builder()
+ .pathStyleAccessEnabled(config.endpoint() != null)
+ .build())
+ .credentialsProvider(
+ StaticCredentialsProvider.create(
+ AwsBasicCredentials.create(
+ config.accessKeyId(), config.secretAccessKey())));
+ if (config.endpoint() != null) {
+ builder.endpointOverride(URI.create(config.endpoint()));
+ }
+ return builder.build();
+ }
+
+ @PreDestroy
+ void closeClients() {
+ clients.values().forEach(S3Client::close);
+ clients.clear();
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Identities.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Identities.java
new file mode 100644
index 0000000000..ae2d279b9f
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Identities.java
@@ -0,0 +1,28 @@
+package stirling.software.proprietary.policy.s3;
+
+import java.time.Instant;
+
+/**
+ * The S3 backend's identity and version scheme, shared by {@code S3InputSource} and {@code
+ * S3OutputSink} so outputs are recorded under exactly the identity and gate the next listing
+ * derives. Identity is {@code s3://bucket/key}; the gate is the ETag every listing returns for free
+ * (multipart ETags are not content hashes, so any ETag change simply reads as a new version).
+ */
+public final class S3Identities {
+
+ private S3Identities() {}
+
+ public static String identity(String bucket, String key) {
+ return "s3://" + bucket + "/" + key;
+ }
+
+ /** ETag stripped of its quotes; falls back to size:lastModified for stores that omit it. */
+ public static String gate(String eTag, Long size, Instant lastModified) {
+ if (eTag != null && !eTag.isBlank()) {
+ return eTag.replace("\"", "");
+ }
+ return (size == null ? -1 : size)
+ + ":"
+ + (lastModified == null ? 0 : lastModified.toEpochMilli());
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java
new file mode 100644
index 0000000000..3a7a73df5b
--- /dev/null
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java
@@ -0,0 +1,27 @@
+package stirling.software.proprietary.policy.source;
+
+/**
+ * The Editor as a virtual, always-present source. Unlike a persisted {@link Source} it is neither
+ * stored nor configurable: it stands for the documents a team processes by running policies from
+ * the in-app editor (the {@code POST /api/v1/policies/{id}/run} path). Its throughput is tracked
+ * through {@link SourceDocCounter} under a synthetic, team-scoped key, so each team sees only its
+ * own editor activity and the client is only ever handed the opaque {@link #ID}, never a team.
+ */
+public final class EditorSource {
+
+ /** The single, stable id and type the client sees for the editor row. */
+ public static final String ID = "editor";
+
+ public static final String TYPE = "editor";
+
+ private EditorSource() {}
+
+ /**
+ * The per-team {@link SourceDocCounter} key. A {@code null} team (login disabled / self-hosted
+ * single user) shares one global bucket; otherwise counts are partitioned by team so a team's
+ * total aggregates every member's editor runs and no other team's.
+ */
+ public static String counterKey(Long teamId) {
+ return teamId == null ? ID : ID + ":" + teamId;
+ }
+}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java
index 1db5663fd2..f5a14282c9 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java
@@ -34,6 +34,13 @@ public class SourceAccessGuard {
/** Team a new source is stamped with: the creator's team. {@code null} when login disabled. */
public Long teamForNewSource() {
+ return currentTeamId();
+ }
+
+ /**
+ * The current user's team (what scopes their sources), or {@code null} when login is disabled.
+ */
+ public Long currentTeamId() {
return enforced() ? policyManagementAuthority.currentUserTeamId() : null;
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java
index 27d60703de..524b8c369c 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java
@@ -1,6 +1,7 @@
package stirling.software.proprietary.policy.source;
import java.util.List;
+import java.util.Map;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.http.HttpStatus;
@@ -29,6 +30,7 @@ import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
+import stirling.software.proprietary.util.SecretMasker;
/**
* CRUD for persisted, reusable input connections plus the Sources overview for the admin portal. A
@@ -65,11 +67,16 @@ public class SourceController {
}
@GetMapping("/{sourceId}")
- @Operation(summary = "Get a source by id")
+ @Operation(
+ summary = "Get a source by id",
+ description =
+ "Secret-bearing options are returned as a redaction sentinel, never their"
+ + " stored values; an edit that sends the sentinel back keeps them.")
public ResponseEntity get(@PathVariable String sourceId) {
return sourceStore
.get(sourceId)
.filter(sourceAccessGuard::canAccess)
+ .map(SourceController::withMaskedSecrets)
.map(ResponseEntity::ok)
.orElseGet(() -> ResponseEntity.notFound().build());
}
@@ -81,6 +88,10 @@ public class SourceController {
"The trailing 30-day per-day document series (oldest first) for the source's"
+ " sparkline.")
public ResponseEntity> documentCounts(@PathVariable String sourceId) {
+ // The editor is virtual: its series is tracked per team, not against a persisted source.
+ if (EditorSource.ID.equals(sourceId)) {
+ return ResponseEntity.ok(overviewService.editorDailySeries());
+ }
return sourceStore
.get(sourceId)
.filter(sourceAccessGuard::canAccess)
@@ -97,7 +108,8 @@ public class SourceController {
+ " matching source type.")
public ResponseEntity save(@RequestBody Source source) {
requireSourceEditingAllowed();
- Source owned = resolveOwnership(source);
+ requireNotEditor(source.id(), source.type());
+ Source owned = withStoredSecrets(resolveOwnership(source));
try {
validateConfig(owned);
} catch (IllegalArgumentException e) {
@@ -107,7 +119,7 @@ public class SourceController {
// An edited folder source can change which directory needs watching, so re-sync trigger
// registrations now instead of waiting for the next reconcile.
policyTriggerManager.notifyPoliciesChanged();
- return ResponseEntity.ok(saved);
+ return ResponseEntity.ok(withMaskedSecrets(saved));
}
@DeleteMapping("/{sourceId}")
@@ -118,6 +130,7 @@ public class SourceController {
+ " so the connection can't be pulled out from under a live policy.")
public ResponseEntity delete(@PathVariable String sourceId) {
requireSourceEditingAllowed();
+ requireNotEditor(sourceId, null);
Source source = sourceStore.get(sourceId).filter(sourceAccessGuard::canAccess).orElse(null);
if (source == null) {
return ResponseEntity.notFound().build();
@@ -169,6 +182,42 @@ public class SourceController {
teamId);
}
+ private static Source withOptions(Source source, Map options) {
+ return new Source(
+ source.id(),
+ source.name(),
+ source.type(),
+ options,
+ source.enabled(),
+ source.owner(),
+ source.teamId());
+ }
+
+ /** Secrets never leave the server: reads return the redaction sentinel in their place. */
+ private static Source withMaskedSecrets(Source source) {
+ return withOptions(source, SecretMasker.mask(source.options()));
+ }
+
+ /**
+ * An edit that round-trips a masked read sends secrets back as the sentinel; restore them from
+ * the stored source so saving without re-typing keeps them (validation then runs against the
+ * real values).
+ */
+ private Source withStoredSecrets(Source incoming) {
+ if (incoming.id() == null || incoming.id().isBlank()) {
+ return incoming;
+ }
+ return sourceStore
+ .get(incoming.id())
+ .map(
+ existing ->
+ withOptions(
+ incoming,
+ SecretMasker.restoreRedacted(
+ incoming.options(), existing.options())))
+ .orElse(incoming);
+ }
+
/** Validate the config against the bean that handles the source's type, as the engine will. */
private void validateConfig(Source source) {
InputSpec spec = source.toInputSpec();
@@ -195,6 +244,18 @@ public class SourceController {
}
}
+ /**
+ * The editor is a built-in, virtual source: it is always present and cannot be created, edited,
+ * or deleted like a persisted connection. Reject any attempt to touch it by id or type.
+ */
+ private static void requireNotEditor(String id, String type) {
+ if (EditorSource.ID.equals(id) || EditorSource.TYPE.equals(type)) {
+ throw new ResponseStatusException(
+ HttpStatus.BAD_REQUEST,
+ "The editor is a built-in source and cannot be created, edited, or deleted");
+ }
+ }
+
/** Names of the caller's visible policies that reference the given source. */
private List referencingPolicyNames(String sourceId) {
return policyAccessGuard.visibleFrom(policyStore).stream()
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceEntity.java
index 412f862f89..c94e4cb079 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceEntity.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceEntity.java
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.source;
import java.io.Serializable;
import jakarta.persistence.Column;
+import jakarta.persistence.Convert;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
@@ -11,6 +12,8 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import stirling.software.proprietary.integration.crypto.LenientEncryptedStringConverter;
+
/**
* JPA row for a {@link Source}. The whole source lives as JSON in {@code sourceJson} (authoritative
* on read); the scalar columns are denormalized copies for querying. {@code owner} and {@code
@@ -45,6 +48,9 @@ public class SourceEntity implements Serializable {
@Column(name = "enabled")
private boolean enabled;
+ // Encrypted at rest: source options carry user-supplied credentials (e.g. an S3 secret
+ // access key). Lenient so rows written before encryption shipped still load.
+ @Convert(converter = LenientEncryptedStringConverter.class)
@Column(name = "source_json", columnDefinition = "text")
private String sourceJson;
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java
index 6e5f92ce44..f202a30e23 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java
@@ -14,6 +14,7 @@ import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.store.PolicyStore;
+import stirling.software.proprietary.util.SecretMasker;
/**
* Builds the Sources overview: every persisted source the caller's team owns, shown exactly once,
@@ -40,7 +41,7 @@ public class SourceOverviewService {
Map docStats =
docCounter.statsFor(sources.stream().map(Source::id).toList());
- List views =
+ List persisted =
sources.stream()
.map(
source ->
@@ -55,7 +56,13 @@ public class SourceOverviewService {
.thenComparing(SourceView::name))
.toList();
- return new SourcesResponse(buildKpis(views), views);
+ // The editor is a built-in source: always present and pinned first. The KPI strip counts
+ // only the connections a team configures, so the editor is left out of the KPIs.
+ List views = new ArrayList<>();
+ views.add(editorView(policies));
+ views.addAll(persisted);
+
+ return new SourcesResponse(buildKpis(persisted), views);
}
/**
@@ -66,6 +73,49 @@ public class SourceOverviewService {
return docCounter.dailySeriesFor(sourceId);
}
+ /** The 30-day daily editor document series (oldest first) for the caller's team. */
+ public List editorDailySeries() {
+ return docCounter.dailySeriesFor(
+ EditorSource.counterKey(sourceAccessGuard.currentTeamId()));
+ }
+
+ /**
+ * The always-present editor row. It has no stored config; its documents are those the team has
+ * processed by running policies from the editor, and it is "used by" every policy that targets
+ * the editor as its source.
+ */
+ private SourceView editorView(List policies) {
+ String key = EditorSource.counterKey(sourceAccessGuard.currentTeamId());
+ DocStats docs = docCounter.statsFor(List.of(key)).getOrDefault(key, DocStats.ZERO);
+ List refs =
+ policies.stream()
+ .filter(SourceOverviewService::runsFromEditor)
+ .map(policy -> new SourceView.PolicyRef(policy.id(), policy.name()))
+ .toList();
+ return new SourceView(
+ EditorSource.ID,
+ "Editor",
+ EditorSource.TYPE,
+ "active",
+ refs.size(),
+ refs,
+ List.of(),
+ docs.total(),
+ docs.last24h(),
+ docs.last30d());
+ }
+
+ /**
+ * Whether a policy runs from the editor. Editor membership is carried in the policy's output
+ * metadata ({@code output.options.sources}) - a client-side list the editor writes when a
+ * policy targets it - rather than as a persisted {@code sourceId}, because the editor is
+ * virtual and has no stored source to reference.
+ */
+ private static boolean runsFromEditor(Policy policy) {
+ Object sources = policy.output().options().get("sources");
+ return sources instanceof List> list && list.contains(EditorSource.ID);
+ }
+
/** Policies referencing each source id, across the caller's visible policies. */
private static Map> referencesBySource(List policies) {
Map> bySource = new HashMap<>();
@@ -104,13 +154,18 @@ public class SourceOverviewService {
return referenceCount == 0 ? "unused" : "active";
}
- /** Generic key/value view of the source's config - works for any source type. */
+ /**
+ * Generic key/value view of the source's config - works for any source type. Secret-bearing
+ * options (e.g. an S3 secret access key) are redacted, not omitted, so the overview still shows
+ * that a credential is configured.
+ */
private static List configRows(Source source) {
- return source.options().entrySet().stream()
+ Map masked = SecretMasker.mask(source.options());
+ return source.options().keySet().stream()
.map(
- entry ->
+ key ->
new SourceView.DetailRow(
- humanize(entry.getKey()), String.valueOf(entry.getValue())))
+ humanize(key), String.valueOf(masked.get(key))))
.toList();
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java
index 08e6bd4fbe..e393e47833 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.store;
import java.io.Serializable;
import jakarta.persistence.Column;
+import jakarta.persistence.Convert;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
@@ -11,6 +12,8 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import stirling.software.proprietary.integration.crypto.LenientEncryptedStringConverter;
+
/**
* JPA row for a {@link stirling.software.proprietary.policy.model.Policy}. The whole policy lives
* as JSON in {@code policyJson} (authoritative on read); the scalar columns are denormalized copies
@@ -55,6 +58,9 @@ public class PolicyEntity implements Serializable {
@Column(name = "sort_order")
private Integer sortOrder;
+ // Encrypted at rest: output options carry user-supplied credentials (e.g. an S3 secret
+ // access key). Lenient so rows written before encryption shipped still load.
+ @Convert(converter = LenientEncryptedStringConverter.class)
@Column(name = "policy_json", columnDefinition = "text")
private String policyJson;
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java
index 3e747d7169..32ba4ec77d 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java
@@ -99,11 +99,17 @@ public class ScheduleTrigger implements PolicyTrigger {
// Baseline a newly-seen policy to now so it does not fire immediately.
Instant last = lastFiredByPolicy.computeIfAbsent(policy.id(), id -> now);
ZonedDateTime next = config.schedule().nextAfter(last.atZone(config.zone()));
- if (!next.toInstant().isAfter(now)) {
- lastFiredByPolicy.put(policy.id(), now);
- log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name());
- policyRunner.run(policy);
+ if (next.toInstant().isAfter(now)) {
+ continue;
}
+ ZonedDateTime later = config.schedule().nextAfter(next);
+ while (!later.toInstant().isAfter(now)) {
+ next = later;
+ later = config.schedule().nextAfter(later);
+ }
+ lastFiredByPolicy.put(policy.id(), next.toInstant());
+ log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name());
+ policyRunner.run(policy);
}
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java
index 7c0081dd7c..e140b8f5c8 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java
@@ -184,6 +184,20 @@ public interface PersistentAuditEventRepository extends JpaRepository findByPrincipalIn(
@Param("principals") List principals, Pageable pageable);
+ // Noise-excluding scans for the portal: keep the page window full of meaningful events so the
+ // list doesn't shrink as read/polling noise (UI_DATA/HTTP_REQUEST) grows in the recent window.
+ @Query("SELECT e FROM PersistentAuditEvent e WHERE e.type NOT IN :excludedTypes")
+ Page findByTypeNotIn(
+ @Param("excludedTypes") List excludedTypes, Pageable pageable);
+
+ @Query(
+ "SELECT e FROM PersistentAuditEvent e WHERE e.type NOT IN :excludedTypes AND e.principal"
+ + " IN :principals")
+ Page findByTypeNotInAndPrincipalIn(
+ @Param("excludedTypes") List excludedTypes,
+ @Param("principals") List principals,
+ Pageable pageable);
+
@Query(
"SELECT e FROM PersistentAuditEvent e WHERE e.type IN :types AND e.timestamp BETWEEN :startDate AND :endDate")
Page findByTypeInAndTimestampBetween(
@@ -276,4 +290,25 @@ public interface PersistentAuditEventRepository extends JpaRepository :since")
+ long countByTypeInAndSourceAndPrincipalInAndTimestampAfter(
+ @Param("types") List types,
+ @Param("source") String source,
+ @Param("principals") List principals,
+ @Param("since") Instant since);
+
+ @Query(
+ "SELECT COUNT(DISTINCT e.principal) FROM PersistentAuditEvent e "
+ + "WHERE e.source = :source AND e.type <> :excludeType "
+ + "AND e.principal IN :principals AND e.timestamp > :since")
+ long countDistinctPrincipalsBySourceExcludingTypeAndPrincipalInAfter(
+ @Param("source") String source,
+ @Param("excludeType") String excludeType,
+ @Param("principals") List principals,
+ @Param("since") Instant since);
}
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java
index 0e9460eca7..74e68d4602 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java
@@ -7,6 +7,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.io.FilenameUtils;
@@ -27,6 +28,7 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.service.AutomationRunContext;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.InternalApiTimeoutException;
@@ -156,33 +158,41 @@ public class AiWorkflowService {
throws IOException {
validateRequest(request);
- // Key by opaque file id, not filename. Filenames aren't guaranteed unique across an
- // upload (users can rotate the same 'scan.pdf' twice), and the engine identifies files
- // by id in every response shape that asks Java to look a file up again.
- Map filesById = new LinkedHashMap<>();
- List files = new ArrayList<>();
- for (AiWorkflowFileInput fileInput : request.getFileInputs()) {
- MultipartFile multipartFile = fileInput.getFileInput();
- AiFile aiFile =
- new AiFile(
- fileIdStrategy.idFor(multipartFile),
- multipartFile.getOriginalFilename());
- filesById.put(aiFile.getId(), multipartFile);
- files.add(aiFile);
- }
+ // One AI orchestration = one automation run. Scope a run id (on whichever thread runs
+ // orchestrate — request thread for sync, stream-executor for streaming) so every tool
+ // sub-step it dispatches via PolicyExecutor → InternalApiClient groups into one charge.
+ try (AutomationRunContext.Scope ignored =
+ AutomationRunContext.open(UUID.randomUUID().toString())) {
- WorkflowTurnRequest initialRequest = new WorkflowTurnRequest();
- initialRequest.setUserMessage(request.getUserMessage().trim());
- initialRequest.setFiles(files);
- initialRequest.setConversationHistory(new ArrayList<>(request.getConversationHistory()));
- initialRequest.setEnabledEndpoints(endpointResolver.getEnabledEndpointUrls());
- listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.ANALYZING));
+ // Key by opaque file id, not filename. Filenames aren't guaranteed unique across an
+ // upload (users can rotate the same 'scan.pdf' twice), and the engine identifies files
+ // by id in every response shape that asks Java to look a file up again.
+ Map filesById = new LinkedHashMap<>();
+ List files = new ArrayList<>();
+ for (AiWorkflowFileInput fileInput : request.getFileInputs()) {
+ MultipartFile multipartFile = fileInput.getFileInput();
+ AiFile aiFile =
+ new AiFile(
+ fileIdStrategy.idFor(multipartFile),
+ multipartFile.getOriginalFilename());
+ filesById.put(aiFile.getId(), multipartFile);
+ files.add(aiFile);
+ }
- WorkflowState state = new WorkflowState.Pending(initialRequest);
- while (state instanceof WorkflowState.Pending pending) {
- state = advance(pending.request(), filesById, listener);
+ WorkflowTurnRequest initialRequest = new WorkflowTurnRequest();
+ initialRequest.setUserMessage(request.getUserMessage().trim());
+ initialRequest.setFiles(files);
+ initialRequest.setConversationHistory(
+ new ArrayList<>(request.getConversationHistory()));
+ initialRequest.setEnabledEndpoints(endpointResolver.getEnabledEndpointUrls());
+ listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.ANALYZING));
+
+ WorkflowState state = new WorkflowState.Pending(initialRequest);
+ while (state instanceof WorkflowState.Pending pending) {
+ state = advance(pending.request(), filesById, listener);
+ }
+ return ((WorkflowState.Terminal) state).response();
}
- return ((WorkflowState.Terminal) state).response();
}
private WorkflowState advance(
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AuditService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AuditService.java
index a56ece5285..2fee8b5c7c 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AuditService.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AuditService.java
@@ -37,9 +37,11 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.service.CustomPDFDocumentFactory;
+import stirling.software.common.service.InternalApiClient;
import stirling.software.common.util.RegexPatternUtils;
import stirling.software.common.util.RequestUriUtils;
import stirling.software.proprietary.accountlink.BillableOperationClassifier;
+import stirling.software.proprietary.audit.AuditContext;
import stirling.software.proprietary.audit.AuditEventType;
import stirling.software.proprietary.audit.AuditLevel;
import stirling.software.proprietary.audit.Audited;
@@ -469,6 +471,73 @@ public class AuditService {
}
}
+ /**
+ * Merge controller-supplied context into the audit data, read in the aspect's {@code finally}
+ * after the controller body has run. A policy run stamps its name and step endpoints as request
+ * attributes ({@link AuditContext}); the internal loopback dispatch that executes each pipeline
+ * step carries the automation marker header ({@link InternalApiClient#AUTOMATION_HEADER}).
+ * Surfacing these lets the portal label a run as its policy instead of the raw {@code /run}
+ * endpoint, and flag its sub-steps as automation rather than direct user actions.
+ *
+ * @param data The existing audit data map
+ * @param req The current request, or null when not in a web context
+ */
+ public void addAutomationContext(Map data, HttpServletRequest req) {
+ if (req == null) {
+ return;
+ }
+ Object policyName = req.getAttribute(AuditContext.REQ_ATTR_POLICY_NAME);
+ if (policyName == null) {
+ // A pipeline step's loopback dispatch carries its parent policy name as a header, so
+ // the
+ // step audit ties back to the policy that ran it (not a bare direct call).
+ String header = req.getHeader(InternalApiClient.POLICY_NAME_HEADER);
+ if (header != null && !header.isBlank()) {
+ policyName = header;
+ }
+ }
+ if (policyName != null) {
+ // Both sources are caller-controlled (the header is spoofable and, unlike the sender in
+ // InternalApiClient, uncapped); strip newlines and cap before persisting to audit JSON.
+ String safe = capLabel(String.valueOf(policyName));
+ if (!safe.isEmpty()) {
+ data.put("policyName", safe);
+ }
+ }
+ Object steps = req.getAttribute(AuditContext.REQ_ATTR_POLICY_STEPS);
+ if (steps instanceof List> list && !list.isEmpty()) {
+ // Caller-supplied and unbounded; cap count and each entry so a crafted run can't
+ // bloat the audit JSON the portal cache loads and parses in bulk.
+ List safeSteps =
+ list.stream()
+ .limit(MAX_POLICY_STEPS)
+ .map(s -> capLabel(String.valueOf(s)))
+ .filter(s -> !s.isEmpty())
+ .toList();
+ if (!safeSteps.isEmpty()) {
+ data.put("policySteps", safeSteps);
+ }
+ }
+ if ("true".equalsIgnoreCase(req.getHeader(InternalApiClient.AUTOMATION_HEADER))) {
+ data.put("automation", Boolean.TRUE);
+ }
+ }
+
+ /** Max characters kept for a persisted policy label; mirrors the InternalApiClient send cap. */
+ private static final int MAX_LABEL_LEN = 200;
+
+ /** Max step endpoints kept on a run's audit event; the portal only shows the first few. */
+ private static final int MAX_POLICY_STEPS = 50;
+
+ /** Single-line, length-capped label safe to persist to audit JSON and render in the portal. */
+ private static String capLabel(String value) {
+ if (value == null) {
+ return "";
+ }
+ String safe = value.replaceAll("[\\r\\n]", " ").trim();
+ return safe.length() > MAX_LABEL_LEN ? safe.substring(0, MAX_LABEL_LEN) : safe;
+ }
+
/**
* Add method arguments to the audit data
*
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalAuditReadService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalAuditReadService.java
index 1678cd095f..5a94e9d2b1 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalAuditReadService.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalAuditReadService.java
@@ -9,6 +9,7 @@ import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
+import stirling.software.proprietary.audit.AuditEventType;
import stirling.software.proprietary.audit.PortalAuditEventRow;
import stirling.software.proprietary.model.security.PersistentAuditEvent;
import stirling.software.proprietary.repository.PersistentAuditEventRepository;
@@ -24,12 +25,21 @@ public class PortalAuditReadService {
/** Newest rows to scan; each surface filters this down to what it shows. */
private static final int SCAN_LIMIT = 400;
+ /**
+ * Read/polling noise excluded at the query level so the scan window stays full of meaningful
+ * events. Otherwise a busy scope's recent rows fill with these and the visible list shrinks as
+ * traffic grows - the "audit getting smaller over time" a user would see. No portal surface
+ * shows these types anyway (the infra tab and documents feed both drop them).
+ */
+ private static final List NOISE_TYPES =
+ List.of(AuditEventType.UI_DATA.name(), AuditEventType.HTTP_REQUEST.name());
+
private final PersistentAuditEventRepository auditRepository;
/** Recent whole-server events (admins). */
@Cacheable(value = CACHE_NAME, key = "'server'")
public List serverEvents() {
- return toRows(auditRepository.findAll(recentPage()).getContent());
+ return toRows(auditRepository.findByTypeNotIn(NOISE_TYPES, recentPage()).getContent());
}
/** Recent events by the given principals (team scope). Empty principals yield an empty list. */
@@ -38,7 +48,10 @@ public class PortalAuditReadService {
if (principals.isEmpty()) {
return List.of();
}
- return toRows(auditRepository.findByPrincipalIn(principals, recentPage()).getContent());
+ return toRows(
+ auditRepository
+ .findByTypeNotInAndPrincipalIn(NOISE_TYPES, principals, recentPage())
+ .getContent());
}
private static PageRequest recentPage() {
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalDocumentsService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalDocumentsService.java
index 8784d5c5b6..fcae36f61b 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalDocumentsService.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalDocumentsService.java
@@ -61,8 +61,17 @@ public class PortalDocumentsService {
continue;
}
String path = asString(data.get("path"));
- String source = sourceLabel(asString(data.get("__origin")));
- String product = "API integration".equals(source) ? "API" : "Editor";
+ // Pipeline steps run over an internal loopback (API-key auth), so origin alone reads as
+ // "API". The automation marker distinguishes a policy-run step from real API traffic.
+ boolean automation = isAutomation(data);
+ String policyName = asString(data.get("policyName"));
+ String source =
+ automation
+ ? (policyName != null && !policyName.isBlank()
+ ? "Policy: " + policyName
+ : "Policy automation")
+ : sourceLabel(asString(data.get("__origin")));
+ String product = automation ? "Automation" : productLabel(source);
String action = prettyTool(path);
boolean failed = isFailure(data);
Instant ts = event.timestamp();
@@ -174,6 +183,15 @@ public class PortalDocumentsService {
return "Web upload";
}
+ private static String productLabel(String source) {
+ return "API integration".equals(source) ? "API" : "Editor";
+ }
+
+ private static boolean isAutomation(Map data) {
+ Object v = data.get("automation");
+ return Boolean.TRUE.equals(v) || "true".equalsIgnoreCase(String.valueOf(v));
+ }
+
private static String docType(String contentType, String name) {
String ct = contentType == null ? "" : contentType.toLowerCase(Locale.ROOT);
if (ct.contains("pdf") || name.toLowerCase(Locale.ROOT).endsWith(".pdf")) {
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalInfraAuditService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalInfraAuditService.java
index 73d45800a8..c47df3649f 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalInfraAuditService.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PortalInfraAuditService.java
@@ -54,6 +54,7 @@ public class PortalInfraAuditService {
.limit(RETURN_LIMIT)
.toList();
+ int policy = (int) events.stream().filter(e -> "policy".equals(e.getCategory())).count();
int processing =
(int) events.stream().filter(e -> "processing".equals(e.getCategory())).count();
int elevation =
@@ -63,6 +64,7 @@ public class PortalInfraAuditService {
InfraAuditSummary summary =
InfraAuditSummary.builder()
.totalEvents(events.size())
+ .policy(policy)
.processing(processing)
.elevation(elevation)
.config(config)
@@ -84,20 +86,32 @@ public class PortalInfraAuditService {
private InfraAuditEventDto toDto(PortalAuditEventRow event) {
Map data = parseData(event);
String path = asString(data.get("path"));
- String category = categoryFor(event.type(), path);
+ String policyName = asString(data.get("policyName"));
+ boolean automation = isAutomation(data);
+ // Classify a dispatch by its real run-path URI, not a policyName: the latter can be spoofed
+ // via the X-Stirling-Policy-Name header to make a direct call pose as a policy row.
+ boolean policyDispatch = isPolicyRunPath(path) && !automation;
+ // A dispatch is its own "policy" category so the UI badges it as a policy run, not a
+ // generic processing op; its internal steps keep their real tool category.
+ String category = policyDispatch ? "policy" : categoryFor(event.type(), path);
return InfraAuditEventDto.builder()
.id(String.valueOf(event.id()))
.timestamp(event.timestamp() == null ? "" : TS_FORMAT.format(event.timestamp()))
.category(category)
- .action(actionFor(event.type(), path))
+ .action(actionFor(event.type(), path, policyName, automation))
.actor(event.principal())
- .target(targetFor(category, path, data))
+ .target(targetFor(category, path, data, policyDispatch))
.status(statusFor(event.type(), category, data))
.latencyMs(asLong(data.get("latencyMs")))
.build();
}
+ private static boolean isAutomation(Map data) {
+ Object v = data.get("automation");
+ return Boolean.TRUE.equals(v) || "true".equalsIgnoreCase(String.valueOf(v));
+ }
+
private Map parseData(PortalAuditEventRow event) {
if (event.data() == null || event.data().isEmpty()) {
return Map.of();
@@ -140,7 +154,37 @@ public class PortalInfraAuditService {
|| p.contains("redact");
}
- private static String actionFor(String type, String path) {
+ /**
+ * Label for a row. A genuine policy dispatch (a {@code /policies/.../run} request) shows the
+ * policy it ran; an internal pipeline step (automation marker) is flagged so it isn't read as a
+ * direct action. The name is only shown as the action on a real run URI, so a spoofed
+ * X-Stirling-Policy-Name header on a direct tool call can't overwrite its true action.
+ */
+ private static String actionFor(
+ String type, String path, String policyName, boolean automation) {
+ if (!automation && isPolicyRunPath(path)) {
+ // A run with no name (ad-hoc pipeline) still reads better than the "run" endpoint.
+ return policyName != null ? policyName : "Policy run";
+ }
+ String base = baseActionFor(type, path);
+ if (automation) {
+ return policyName != null
+ ? base + " (policy: " + policyName + ")"
+ : base + " (automation)";
+ }
+ return base;
+ }
+
+ /**
+ * The pipeline-run endpoints: {@code /policies/run}, {@code /run/stream}, {@code /{id}/run}.
+ */
+ private static boolean isPolicyRunPath(String path) {
+ return path != null
+ && path.contains("/policies/")
+ && (path.endsWith("/run") || path.endsWith("/run/stream"));
+ }
+
+ private static String baseActionFor(String type, String path) {
AuditEventType t = AuditEventType.fromString(type);
if (t == null) {
return prettyTool(path);
@@ -191,7 +235,32 @@ public class PortalInfraAuditService {
return sb.isEmpty() ? "PDF operation" : sb.toString();
}
- private static String targetFor(String category, String path, Map data) {
+ /** "Auto Redact, Compress PDF" from the run's step endpoints; first three, then "+N more". */
+ private static String prettyStepList(Object steps) {
+ if (!(steps instanceof List> list) || list.isEmpty()) {
+ return null;
+ }
+ int shown = Math.min(3, list.size());
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < shown; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(prettyTool(asString(list.get(i))));
+ }
+ if (list.size() > shown) {
+ sb.append(" +").append(list.size() - shown).append(" more");
+ }
+ return sb.toString();
+ }
+
+ private static String targetFor(
+ String category, String path, Map data, boolean policyDispatch) {
+ if (policyDispatch) {
+ // The run touches no single file at this level; show the tools the policy runs instead.
+ String steps = prettyStepList(data.get("policySteps"));
+ return steps != null ? steps : "Pipeline";
+ }
if ("auth".equals(category)) {
// Auth events don't act on a resource; the session is the closest thing.
return "Web session";
diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java b/app/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java
index a3975de194..5a9619c2dd 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java
@@ -1,5 +1,6 @@
package stirling.software.proprietary.util;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
@@ -13,10 +14,15 @@ import stirling.software.common.util.RegexPatternUtils;
@Slf4j
public final class SecretMasker {
+ /** The placeholder masked values are replaced with; reads as "a secret is set". */
+ public static final String REDACTED = "********";
+
private static final Pattern SENSITIVE =
RegexPatternUtils.getInstance()
.getPattern(
- "(?i)\\b(password|token|secret|api[_-]?key|authorization|auth|jwt|cred|cert)\\b");
+ // secret[_-]?access[_-]?key precedes plain secret so camelCase keys
+ // like secretAccessKey (no word boundary after "secret") still match.
+ "(?i)\\b(password|token|secret[_-]?access[_-]?key|secret|api[_-]?key|authorization|auth|jwt|cred|cert)\\b");
private SecretMasker() {}
@@ -47,8 +53,28 @@ public final class SecretMasker {
private static Object deepMaskValue(String key, Object value) {
if (key != null && SENSITIVE.matcher(key).find()) {
- return "***REDACTED***";
+ return REDACTED;
}
return deepMask(value);
}
+
+ /**
+ * Restore top-level values the caller sent back as the {@link #REDACTED} sentinel from the
+ * stored map, so a masked read can round-trip through an edit without re-typing secrets. A
+ * sentinel with no stored counterpart is left as-is (it fails whatever validates it, rather
+ * than silently passing an unset secret).
+ */
+ public static Map restoreRedacted(
+ Map incoming, Map stored) {
+ if (incoming == null || stored == null) {
+ return incoming;
+ }
+ Map merged = new LinkedHashMap<>(incoming);
+ merged.replaceAll(
+ (key, value) ->
+ REDACTED.equals(value) && stored.containsKey(key)
+ ? stored.get(key)
+ : value);
+ return merged;
+ }
}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java
index baa6755493..c63e6b8e47 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java
@@ -2,6 +2,7 @@ package stirling.software.proprietary.access.service;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import java.lang.reflect.Field;
@@ -258,6 +259,24 @@ class ResourceAccessServiceTest {
.isFalse();
}
+ @Test
+ void teamLeadDefaultOnTeamResourceDeniesForeignTeamsLead() {
+ // Team-owned resource: the default admits only the owning team's leads. A lead of
+ // some other team must be denied even though an unscoped "is any team leader"
+ // check would admit them (lenient stub: the scoped path must never consult it).
+ stubGrants();
+ User foreignLead = user(6);
+ lenient().when(teamLeadLookup.isAnyTeamLeader(foreignLead)).thenReturn(true);
+ assertThat(
+ service.canUseResource(
+ TYPE,
+ RID,
+ PrincipalRef.team(7L),
+ DefaultAccessPolicy.ADMINS_AND_TEAM_LEADS,
+ foreignLead))
+ .isFalse();
+ }
+
// ---- portal convenience (default policy ADMINS_AND_TEAM_LEADS) ----
@Test
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/integration/crypto/LenientEncryptedStringConverterTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/integration/crypto/LenientEncryptedStringConverterTest.java
new file mode 100644
index 0000000000..b61ded9708
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/integration/crypto/LenientEncryptedStringConverterTest.java
@@ -0,0 +1,45 @@
+package stirling.software.proprietary.integration.crypto;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class LenientEncryptedStringConverterTest {
+
+ private final LenientEncryptedStringConverter converter = new LenientEncryptedStringConverter();
+
+ @BeforeAll
+ static void initKey() throws Exception {
+ KeyGenerator generator = KeyGenerator.getInstance("AES");
+ generator.init(256);
+ SecretKey key = generator.generateKey();
+ CredentialEncryption.initialiseForTesting(key);
+ }
+
+ @Test
+ void roundTripsThroughCiphertext() {
+ String json = "{\"bucket\":\"inbox\",\"secretAccessKey\":\"shh\"}";
+
+ String stored = converter.convertToDatabaseColumn(json);
+
+ assertThat(stored).isNotEqualTo(json).doesNotContain("shh");
+ assertThat(converter.convertToEntityAttribute(stored)).isEqualTo(json);
+ }
+
+ @Test
+ void legacyPlaintextRowsPassThroughOnRead() {
+ String legacy = "{\"bucket\":\"inbox\",\"mode\":\"consume\"}";
+
+ assertThat(converter.convertToEntityAttribute(legacy)).isEqualTo(legacy);
+ }
+
+ @Test
+ void nullsPassThrough() {
+ assertThat(converter.convertToDatabaseColumn(null)).isNull();
+ assertThat(converter.convertToEntityAttribute(null)).isNull();
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java
index f38263ad3e..c02945fdad 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java
@@ -9,6 +9,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@@ -17,6 +18,7 @@ import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpStatus;
@@ -34,16 +36,21 @@ import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.PolicyValidator;
+import stirling.software.proprietary.policy.engine.SweepOutcome;
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
+import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunView;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
+import stirling.software.proprietary.policy.source.EditorSource;
import stirling.software.proprietary.policy.source.SourceAccessGuard;
+import stirling.software.proprietary.policy.source.SourceDocCounter;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
+import stirling.software.proprietary.util.SecretMasker;
@ExtendWith(MockitoExtension.class)
@DisplayName("PolicyController")
@@ -54,6 +61,7 @@ class PolicyControllerTest {
@Mock private stirling.software.proprietary.policy.store.PolicyStore policyStore;
@Mock private SourceStore sourceStore;
@Mock private SourceAccessGuard sourceAccessGuard;
+ @Mock private SourceDocCounter docCounter;
@Mock private PolicyValidator policyValidator;
@Mock private PolicyAccessGuard policyAccessGuard;
@Mock private PolicyManagementAuthority policyManagementAuthority;
@@ -87,6 +95,7 @@ class PolicyControllerTest {
policyStore,
sourceStore,
sourceAccessGuard,
+ docCounter,
policyValidator,
policyAccessGuard,
policyManagementAuthority,
@@ -128,6 +137,17 @@ class PolicyControllerTest {
return new Policy(id, "name", "owner", true, null, List.of(), List.of(), null, teamId);
}
+ private static Policy s3OutputPolicy(String id, String secret) {
+ OutputSpec output =
+ new OutputSpec(
+ "s3",
+ Map.of(
+ "bucket", "outbox",
+ "accessKeyId", "AKIAEXAMPLE",
+ "secretAccessKey", secret));
+ return new Policy(id, "name", "owner", true, null, List.of(), List.of(), output, 1L);
+ }
+
private static PolicyRunHandle handle(String runId) {
PolicyRun run = new PolicyRun(runId, null, definitionWithStep());
return new PolicyRunHandle(runId, CompletableFuture.completedFuture(run));
@@ -150,6 +170,18 @@ class PolicyControllerTest {
assertThat(response.getBody().getJobId()).isEqualTo("run-1");
}
+ @Test
+ @DisplayName("feeds the editor source, scoped to the caller's team")
+ void adHocRunFeedsTheEditorSource() throws Exception {
+ when(policyRunner.runAdHoc(any(), any(), eq(PolicyProgressListener.NOOP)))
+ .thenReturn(handle("run-1"));
+ when(sourceAccessGuard.currentTeamId()).thenReturn(3L);
+
+ controller.run(definitionWithStep(), new PolicyRunFiles());
+
+ verify(docCounter).record(EditorSource.counterKey(3L), 0L);
+ }
+
@Test
@DisplayName("rejects a pipeline with no steps")
void rejectsEmptyPipeline() {
@@ -263,6 +295,27 @@ class PolicyControllerTest {
verify(policyTriggerManager).notifyPoliciesChanged();
}
+ @Test
+ @DisplayName("saving the sentinel back keeps the stored output secret")
+ void saveRestoresOutputSecrets() {
+ applicationProperties.getSecurity().setEnableLogin(false);
+ Policy existing = s3OutputPolicy("p1", "shh");
+ when(policyStore.get("p1")).thenReturn(Optional.of(existing));
+ when(policyAccessGuard.canAccess(existing)).thenReturn(true);
+ when(policyStore.save(any())).thenAnswer(i -> i.getArgument(0));
+
+ ResponseEntity response =
+ controller.savePolicy(s3OutputPolicy("p1", SecretMasker.REDACTED));
+
+ ArgumentCaptor stored = ArgumentCaptor.forClass(Policy.class);
+ verify(policyStore).save(stored.capture());
+ assertThat(stored.getValue().output().options().get("secretAccessKey"))
+ .isEqualTo("shh");
+ // The save response is masked again; only the store sees the real value.
+ assertThat(response.getBody().output().options().get("secretAccessKey"))
+ .isEqualTo(SecretMasker.REDACTED);
+ }
+
@Test
@DisplayName("forbidden when login enabled and caller cannot edit")
void forbidden() {
@@ -363,6 +416,20 @@ class PolicyControllerTest {
assertThat(response.getBody().id()).isEqualTo("a");
}
+ @Test
+ @DisplayName("getPolicy returns output secrets as the redaction sentinel")
+ void getMasksOutputSecrets() {
+ Policy p = s3OutputPolicy("a", "shh");
+ when(policyStore.get("a")).thenReturn(Optional.of(p));
+ when(policyAccessGuard.canAccess(p)).thenReturn(true);
+
+ Policy read = controller.getPolicy("a").getBody();
+
+ assertThat(read.output().options().get("secretAccessKey"))
+ .isEqualTo(SecretMasker.REDACTED);
+ assertThat(read.output().options().get("bucket")).isEqualTo("outbox");
+ }
+
@Test
@DisplayName("getPolicy returns 404 when not accessible")
void getNotAccessible() {
@@ -536,17 +603,18 @@ class PolicyControllerTest {
}
@Test
- @DisplayName("trigger runs an accessible policy against its sources and returns run ids")
+ @DisplayName("trigger runs an accessible policy against its sources and returns the sweep")
void triggersRun() {
Policy p = policy("a", 1L);
when(policyStore.get("a")).thenReturn(Optional.of(p));
when(policyAccessGuard.canAccess(p)).thenReturn(true);
- when(policyRunner.run(p)).thenReturn(List.of("run-a", "run-b"));
+ SweepOutcome outcome = new SweepOutcome(List.of("run-a", "run-b"), 3, 1, 0, 0);
+ when(policyRunner.run(p)).thenReturn(outcome);
- ResponseEntity> response = controller.trigger("a");
+ ResponseEntity response = controller.trigger("a");
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.ACCEPTED);
- assertThat(response.getBody()).containsExactly("run-a", "run-b");
+ assertThat(response.getBody()).isEqualTo(outcome);
}
@Test
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java
index 13189b2481..e21f371305 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java
@@ -1,5 +1,6 @@
package stirling.software.proprietary.policy.engine;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -26,10 +27,12 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.core.io.ByteArrayResource;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.input.ResolveContext;
import stirling.software.proprietary.policy.input.ResolvedInput;
+import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
@@ -39,6 +42,7 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
+import stirling.software.proprietary.policy.source.EditorSource;
import stirling.software.proprietary.policy.source.InProcessSourceDocCounter;
import stirling.software.proprietary.policy.source.InProcessSourceStore;
import stirling.software.proprietary.policy.source.Source;
@@ -56,6 +60,7 @@ class PolicyRunnerTest {
@Mock private ProcessedLedger processedLedger;
private final SourceStore sourceStore = new InProcessSourceStore();
+ private final InProcessSourceDocCounter docCounter = new InProcessSourceDocCounter();
private PolicyRunner runner;
@BeforeEach
@@ -65,7 +70,7 @@ class PolicyRunnerTest {
policyEngine,
List.of(folderSource),
sourceStore,
- new InProcessSourceDocCounter(),
+ docCounter,
processedLedger);
}
@@ -85,6 +90,42 @@ class PolicyRunnerTest {
verify(processedLedger).deleteUnseen(eq("p1"), anyLong());
}
+ @Test
+ void reportsWhatTheSweepSkippedSoAnEmptyTriggerExplainsItself() throws Exception {
+ InProcessProcessedLedger ledger = new InProcessProcessedLedger();
+ PolicyRunner reporting =
+ new PolicyRunner(
+ policyEngine,
+ List.of(folderSource),
+ sourceStore,
+ new InProcessSourceDocCounter(),
+ ledger);
+ InputSpec spec = InputSpec.folder("/in");
+ Policy policy = policy(List.of(spec));
+ // One file already processed at its current version, one parked by a failed run.
+ ledger.claim("p1", "/in/done.pdf", "g1", null);
+ ledger.settle("p1", "/in/done.pdf", "g1", null, true);
+ ledger.claim("p1", "/in/failed.pdf", "g2", null);
+ ledger.settle("p1", "/in/failed.pdf", "g2", null, false);
+ when(folderSource.supports(spec)).thenReturn(true);
+ when(folderSource.resolve(eq(spec), any()))
+ .thenAnswer(
+ invocation -> {
+ ResolveContext ctx = invocation.getArgument(1);
+ ctx.reportPresent(List.of("/in/done.pdf", "/in/failed.pdf"));
+ // Both are at their settled versions, so neither claims.
+ return List.of();
+ });
+
+ SweepOutcome outcome = reporting.run(policy);
+
+ assertTrue(outcome.runIds().isEmpty());
+ assertEquals(2, outcome.filesListed());
+ assertEquals(1, outcome.alreadyProcessed());
+ assertEquals(1, outcome.parked());
+ assertEquals(0, outcome.inFlight());
+ }
+
@Test
void pullsEverySourceAndRunsOnePerUnitOfWork() throws Exception {
InputSpec spec = InputSpec.folder("/in");
@@ -257,6 +298,33 @@ class PolicyRunnerTest {
verifyNoInteractions(folderSource);
}
+ @Test
+ void runWithRecordsSuppliedDocsAgainstTheEditorSourceForThePolicyTeam() {
+ Policy policy =
+ new Policy(
+ "p1",
+ "p",
+ "owner",
+ true,
+ null,
+ List.of(),
+ List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
+ OutputSpec.inline(),
+ 7L);
+ PolicyInputs inputs =
+ PolicyInputs.of(
+ List.of(
+ new ByteArrayResource("a".getBytes()),
+ new ByteArrayResource("b".getBytes())));
+ when(policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP))
+ .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
+
+ runner.runWith(policy, inputs, PolicyProgressListener.NOOP);
+
+ String key = EditorSource.counterKey(7L);
+ assertEquals(2, docCounter.statsFor(List.of(key)).get(key).total());
+ }
+
/** Persists each spec as a source and returns a policy referencing them by id. */
private Policy policy(List sources) {
List sourceIds =
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/S3InputSourceMinioTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/S3InputSourceMinioTest.java
new file mode 100644
index 0000000000..4e1e1fc305
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/S3InputSourceMinioTest.java
@@ -0,0 +1,231 @@
+package stirling.software.proprietary.policy.input;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.MinIOContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
+import stirling.software.proprietary.policy.model.InputSpec;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+/**
+ * End-to-end {@link S3InputSource} test against a real S3 API (MinIO), through the production
+ * client factory: listing, claiming, streaming, consensus delete, and save-time validation.
+ */
+@Testcontainers(disabledWithoutDocker = true)
+class S3InputSourceMinioTest {
+
+ private static final String POLICY = "p1";
+ private static final String ACCESS_KEY = "minioadmin";
+ private static final String SECRET_KEY = "minioadmin";
+
+ @Container
+ static MinIOContainer minio =
+ new MinIOContainer("minio/minio:latest")
+ .withUserName(ACCESS_KEY)
+ .withPassword(SECRET_KEY);
+
+ private static S3Client adminClient;
+ private static int bucketCounter;
+
+ private String bucket;
+ private S3InputSource source;
+ private InProcessProcessedLedger ledger;
+ private RecordingContext ctx;
+
+ @BeforeEach
+ void setUp() {
+ if (adminClient == null) {
+ adminClient =
+ S3Client.builder()
+ .endpointOverride(java.net.URI.create(minio.getS3URL()))
+ .httpClient(UrlConnectionHttpClient.create())
+ .region(Region.US_EAST_1)
+ .credentialsProvider(
+ StaticCredentialsProvider.create(
+ AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
+ .serviceConfiguration(
+ S3Configuration.builder().pathStyleAccessEnabled(true).build())
+ .build();
+ }
+ bucket = "policy-inbox-" + ++bucketCounter;
+ adminClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
+
+ // The MinIO endpoint resolves to loopback, so the operator opt-in must be on.
+ ApplicationProperties properties = new ApplicationProperties();
+ properties.getPolicies().setAllowPrivateS3Endpoints(true);
+ source = new S3InputSource(new S3ConnectionPool(properties));
+ ledger = new InProcessProcessedLedger();
+ ctx = new RecordingContext();
+ }
+
+ @Test
+ void consumeListsStreamsAndDeletesByConsensus() throws IOException {
+ put("incoming/doc.pdf", "pdf bytes");
+ put("incoming/other.txt", "text");
+
+ List work = source.resolve(spec(Map.of("prefix", "incoming/")), ctx);
+
+ assertThat(work).hasSize(2);
+ assertThat(ctx.present)
+ .containsExactlyInAnyOrder(
+ "s3://" + bucket + "/incoming/doc.pdf",
+ "s3://" + bucket + "/incoming/other.txt");
+ assertThat(read(work.get(0))).isIn("pdf bytes", "text");
+ // In flight: nothing to claim on a second sweep.
+ assertThat(source.resolve(spec(Map.of("prefix", "incoming/")), ctx)).isEmpty();
+
+ work.forEach(unit -> unit.onComplete().accept(true));
+ assertThat(exists("incoming/doc.pdf")).isFalse();
+ assertThat(exists("incoming/other.txt")).isFalse();
+ }
+
+ @Test
+ void aFailedObjectStaysInTheBucket() throws IOException {
+ put("doc.pdf", "data");
+
+ source.resolve(spec(Map.of()), ctx).get(0).onComplete().accept(false);
+
+ assertThat(exists("doc.pdf")).isTrue();
+ assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty();
+ }
+
+ @Test
+ void anObjectOverwrittenMidRunSurvivesTheDeleteAndRunsAgain() throws IOException {
+ put("doc.pdf", "v1");
+
+ List work = source.resolve(spec(Map.of()), ctx);
+ put("doc.pdf", "v2 with a different etag");
+ work.get(0).onComplete().accept(true);
+
+ assertThat(exists("doc.pdf")).isTrue();
+ assertThat(source.resolve(spec(Map.of()), ctx)).hasSize(1);
+ }
+
+ @Test
+ void prefixLimitsWhatIsRead() throws IOException {
+ put("incoming/doc.pdf", "data");
+ put("archive/old.pdf", "data");
+
+ List work = source.resolve(spec(Map.of("prefix", "incoming/")), ctx);
+
+ assertThat(work).hasSize(1);
+ assertThat(ctx.present).containsExactly("s3://" + bucket + "/incoming/doc.pdf");
+ }
+
+ @Test
+ void validateAcceptsAReachableBucketAndRejectsBadCredentials() {
+ source.validate(spec(Map.of()));
+
+ Map wrongSecret = new HashMap<>(baseOptions());
+ wrongSecret.put("secretAccessKey", "not-the-secret");
+ assertThatThrownBy(() -> source.validate(new InputSpec("s3", wrongSecret)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("cannot access");
+
+ Map missingBucket = new HashMap<>(baseOptions());
+ missingBucket.put("bucket", "no-such-bucket-here");
+ assertThatThrownBy(() -> source.validate(new InputSpec("s3", missingBucket)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("cannot access");
+ }
+
+ @Test
+ void aPrivateEndpointIsRejectedWithoutTheOperatorOptIn() {
+ S3InputSource guarded =
+ new S3InputSource(new S3ConnectionPool(new ApplicationProperties()));
+
+ assertThatThrownBy(() -> guarded.validate(spec(Map.of())))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("policies.allowPrivateS3Endpoints");
+ }
+
+ private Map baseOptions() {
+ return Map.of(
+ "bucket", bucket,
+ "endpoint", minio.getS3URL(),
+ "accessKeyId", ACCESS_KEY,
+ "secretAccessKey", SECRET_KEY);
+ }
+
+ private InputSpec spec(Map extra) {
+ Map options = new HashMap<>(baseOptions());
+ options.putAll(extra);
+ return new InputSpec("s3", options);
+ }
+
+ private void put(String key, String content) {
+ adminClient.putObject(
+ PutObjectRequest.builder().bucket(bucket).key(key).build(),
+ RequestBody.fromString(content, StandardCharsets.UTF_8));
+ }
+
+ private boolean exists(String key) {
+ try {
+ adminClient.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build());
+ return true;
+ } catch (NoSuchKeyException e) {
+ return false;
+ }
+ }
+
+ private static String read(ResolvedInput unit) throws IOException {
+ try (InputStream stream = unit.inputs().primary().get(0).getInputStream()) {
+ return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private class RecordingContext implements ResolveContext {
+
+ private final List present = new ArrayList<>();
+
+ @Override
+ public boolean claim(String identity, String gate, Supplier contentHash) {
+ return ledger.claim(POLICY, identity, gate, contentHash);
+ }
+
+ @Override
+ public void settle(
+ String identity, String finalGate, String finalContentHash, boolean success) {
+ ledger.settle(POLICY, identity, finalGate, finalContentHash, success);
+ }
+
+ @Override
+ public boolean allSettledDone(String identity) {
+ return ledger.allSettledDone(identity);
+ }
+
+ @Override
+ public void reportPresent(Collection identities) {
+ present.addAll(identities);
+ }
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/S3InputSourceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/S3InputSourceTest.java
new file mode 100644
index 0000000000..73995248dd
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/S3InputSourceTest.java
@@ -0,0 +1,327 @@
+package stirling.software.proprietary.policy.input;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
+import stirling.software.proprietary.policy.model.InputSpec;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.http.AbortableInputStream;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+/**
+ * Tests for {@link S3InputSource}: consume mode tracks objects in place through the ledger and
+ * removes them by consensus, snapshot stays stateless, and discovery skips folder placeholders and
+ * dot-prefixed keys.
+ */
+@ExtendWith(MockitoExtension.class)
+class S3InputSourceTest {
+
+ private static final String POLICY = "p1";
+ private static final String BUCKET = "inbox-bucket";
+
+ @Mock private S3Client s3Client;
+
+ private S3InputSource source;
+ private InProcessProcessedLedger ledger;
+ private RecordingContext ctx;
+
+ @BeforeEach
+ void setUp() {
+ source =
+ new S3InputSource(
+ new S3ConnectionPool(new ApplicationProperties(), config -> s3Client));
+ ledger = new InProcessProcessedLedger();
+ ctx = new RecordingContext();
+ }
+
+ @Test
+ void consumeRemovesTheObjectOnceProcessed() throws IOException {
+ listingReturns(object("doc.pdf", "\"etag-1\""));
+ headReturns("doc.pdf", "\"etag-1\"");
+
+ List work = source.resolve(spec(), ctx);
+
+ assertEquals(1, work.size());
+ assertEquals(1, work.get(0).inputs().primary().size());
+ // In flight: a second sweep does not pick it up again.
+ assertTrue(source.resolve(spec(), ctx).isEmpty());
+
+ work.get(0).onComplete().accept(true);
+ verify(s3Client).deleteObject(any(DeleteObjectRequest.class));
+ assertTrue(source.resolve(spec(), ctx).isEmpty());
+ }
+
+ @Test
+ void anObjectReplacedMidRunSurvivesTheDelete() throws IOException {
+ listingReturns(object("doc.pdf", "\"etag-1\""));
+ // The object is overwritten while the run is executing.
+ headReturns("doc.pdf", "\"etag-2\"");
+
+ List work = source.resolve(spec(), ctx);
+ work.get(0).onComplete().accept(true);
+
+ // The delete is version-guarded: the replacement is not the object that ran, so it stays
+ // and is claimed as fresh work instead of being marked processed.
+ verify(s3Client, never()).deleteObject(any(DeleteObjectRequest.class));
+ listingReturns(object("doc.pdf", "\"etag-2\""));
+ assertEquals(1, source.resolve(spec(), ctx).size());
+ }
+
+ @Test
+ void aSharedObjectIsRemovedOnlyOnceEveryPolicyHasProcessedIt() throws IOException {
+ listingReturns(object("doc.pdf", "\"etag-1\""));
+ headReturns("doc.pdf", "\"etag-1\"");
+ RecordingContext other = new RecordingContext("p2");
+
+ List mine = source.resolve(spec(), ctx);
+ List theirs = source.resolve(spec(), other);
+ assertEquals(1, mine.size());
+ assertEquals(1, theirs.size());
+
+ mine.get(0).onComplete().accept(true);
+ // The other policy's claim is still in flight, so the first finisher must not delete.
+ verify(s3Client, never()).deleteObject(any(DeleteObjectRequest.class));
+
+ theirs.get(0).onComplete().accept(true);
+ verify(s3Client).deleteObject(any(DeleteObjectRequest.class));
+ }
+
+ @Test
+ void aFailedObjectStaysAndIsNotRetriedUntilItChanges() throws IOException {
+ listingReturns(object("doc.pdf", "\"etag-1\""));
+
+ source.resolve(spec(), ctx).get(0).onComplete().accept(false);
+
+ verify(s3Client, never()).deleteObject(any(DeleteObjectRequest.class));
+ assertTrue(source.resolve(spec(), ctx).isEmpty());
+
+ // A new upload carries a new ETag, which reads as a new version and retries.
+ listingReturns(object("doc.pdf", "\"etag-2\""));
+ assertEquals(1, source.resolve(spec(), ctx).size());
+ }
+
+ @Test
+ void snapshotReadsStatelesslyEverySweep() throws IOException {
+ listingReturns(object("doc.pdf", "\"etag-1\""));
+ InputSpec spec = new InputSpec("s3", options(Map.of("mode", "snapshot")));
+
+ List first = source.resolve(spec, ctx);
+ first.get(0).onComplete().accept(true);
+ List second = source.resolve(spec, ctx);
+
+ assertEquals(1, first.size());
+ assertEquals(1, second.size());
+ verify(s3Client, never()).deleteObject(any(DeleteObjectRequest.class));
+ assertTrue(ctx.present.isEmpty());
+ }
+
+ @Test
+ void folderPlaceholdersAndDotPrefixedKeysAreSkipped() throws IOException {
+ listingReturns(
+ object("doc.pdf", "\"etag-1\""),
+ object("incoming/", "\"etag-2\""),
+ object(".stirling/tmp/staged.pdf", "\"etag-3\""),
+ object("incoming/.hidden.pdf", "\"etag-4\""));
+
+ List work = source.resolve(spec(), ctx);
+
+ assertEquals(1, work.size());
+ assertEquals(List.of("s3://" + BUCKET + "/doc.pdf"), ctx.present);
+ }
+
+ @Test
+ void listingPagesAreAllRead() throws IOException {
+ ListObjectsV2Response firstPage =
+ ListObjectsV2Response.builder()
+ .contents(object("a.pdf", "\"etag-a\""))
+ .nextContinuationToken("next")
+ .build();
+ ListObjectsV2Response secondPage =
+ ListObjectsV2Response.builder().contents(object("b.pdf", "\"etag-b\"")).build();
+ when(s3Client.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenReturn(firstPage, secondPage);
+
+ assertEquals(2, source.resolve(spec(), ctx).size());
+ }
+
+ @Test
+ void aListingFailurePropagatesSoTheSweepVetoesCleanup() {
+ when(s3Client.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenThrow(SdkClientException.create("connection refused"));
+
+ assertThrows(SdkClientException.class, () -> source.resolve(spec(), ctx));
+ }
+
+ @Test
+ void resourceStreamsTheObjectAndNamesItByKeyBasename() throws IOException {
+ listingReturns(object("incoming/doc.pdf", "\"etag-1\""));
+ byte[] payload = "data".getBytes(StandardCharsets.UTF_8);
+ when(s3Client.getObject(any(GetObjectRequest.class)))
+ .thenReturn(
+ new ResponseInputStream<>(
+ GetObjectResponse.builder().build(),
+ AbortableInputStream.create(new ByteArrayInputStream(payload))));
+
+ var resource = source.resolve(spec(), ctx).get(0).inputs().primary().get(0);
+
+ assertEquals("doc.pdf", resource.getFilename());
+ // Content length comes from the listing, not a download.
+ assertEquals(4, resource.contentLength());
+ try (var stream = resource.getInputStream()) {
+ assertEquals("data", new String(stream.readAllBytes(), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ void aMissingETagFallsBackToSizeAndLastModified() throws IOException {
+ Instant modified = Instant.parse("2026-01-01T00:00:00Z");
+ listingReturns(S3Object.builder().key("doc.pdf").size(4L).lastModified(modified).build());
+
+ assertEquals(1, source.resolve(spec(), ctx).size());
+ // The same gate on the next sweep reads as already claimed.
+ listingReturns(S3Object.builder().key("doc.pdf").size(4L).lastModified(modified).build());
+ assertTrue(source.resolve(spec(), ctx).isEmpty());
+ }
+
+ @Test
+ void validateRejectsBadConfig() {
+ // No bucket.
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> source.validate(new InputSpec("s3", Map.of())));
+ // Credentials are required, never the server's own identity - together and individually.
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> source.validate(new InputSpec("s3", Map.of("bucket", BUCKET))));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ source.validate(
+ new InputSpec(
+ "s3", Map.of("bucket", BUCKET, "accessKeyId", "AKIA"))));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> source.validate(new InputSpec("s3", options(Map.of("mode", "sideways")))));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ source.validate(
+ new InputSpec(
+ "s3", options(Map.of("endpoint", "ftp://example.com")))));
+ }
+
+ @Test
+ void validateRejectsAnUnreachableBucket() {
+ when(s3Client.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenThrow(SdkClientException.create("connection refused"));
+
+ assertThrows(IllegalArgumentException.class, () -> source.validate(spec()));
+ }
+
+ private static InputSpec spec() {
+ return new InputSpec("s3", options(Map.of()));
+ }
+
+ /** The required options (bucket + credentials) plus any extras under test. */
+ private static Map options(Map extra) {
+ Map options = new HashMap<>(extra);
+ options.put("bucket", BUCKET);
+ options.put("accessKeyId", "AKIAEXAMPLE");
+ options.put("secretAccessKey", "shh");
+ return options;
+ }
+
+ private static S3Object object(String key, String eTag) {
+ return S3Object.builder()
+ .key(key)
+ .eTag(eTag)
+ .size(4L)
+ .lastModified(Instant.parse("2026-01-01T00:00:00Z"))
+ .build();
+ }
+
+ private void listingReturns(S3Object... objects) {
+ when(s3Client.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenReturn(ListObjectsV2Response.builder().contents(objects).build());
+ }
+
+ private void headReturns(String key, String eTag) {
+ when(s3Client.headObject(any(HeadObjectRequest.class)))
+ .thenReturn(
+ HeadObjectResponse.builder()
+ .eTag(eTag)
+ .contentLength(4L)
+ .lastModified(Instant.parse("2026-01-01T00:00:00Z"))
+ .build());
+ }
+
+ private class RecordingContext implements ResolveContext {
+
+ private final String policyId;
+ private final List present = new ArrayList<>();
+
+ private RecordingContext() {
+ this(POLICY);
+ }
+
+ private RecordingContext(String policyId) {
+ this.policyId = policyId;
+ }
+
+ @Override
+ public boolean claim(String identity, String gate, Supplier contentHash) {
+ return ledger.claim(policyId, identity, gate, contentHash);
+ }
+
+ @Override
+ public void settle(
+ String identity, String finalGate, String finalContentHash, boolean success) {
+ ledger.settle(policyId, identity, finalGate, finalContentHash, success);
+ }
+
+ @Override
+ public boolean allSettledDone(String identity) {
+ return ledger.allSettledDone(identity);
+ }
+
+ @Override
+ public void reportPresent(Collection identities) {
+ present.addAll(identities);
+ }
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkMinioTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkMinioTest.java
new file mode 100644
index 0000000000..a2a5a41ca0
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkMinioTest.java
@@ -0,0 +1,211 @@
+package stirling.software.proprietary.policy.output;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.Resource;
+import org.testcontainers.containers.MinIOContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.job.ResultFile;
+import stirling.software.proprietary.policy.input.ResolveContext;
+import stirling.software.proprietary.policy.input.ResolvedInput;
+import stirling.software.proprietary.policy.input.S3InputSource;
+import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
+import stirling.software.proprietary.policy.model.InputSpec;
+import stirling.software.proprietary.policy.model.OutputSpec;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+/**
+ * End-to-end {@link S3OutputSink} test against a real S3 API (MinIO): uploads, collision renaming,
+ * and - composed with {@link S3InputSource} - the loop-safety guarantee that a policy writing into
+ * a bucket it also watches never re-ingests its own outputs, while a second policy still can.
+ */
+@Testcontainers(disabledWithoutDocker = true)
+class S3OutputSinkMinioTest {
+
+ private static final String POLICY = "p1";
+ private static final String ACCESS_KEY = "minioadmin";
+ private static final String SECRET_KEY = "minioadmin";
+
+ @Container
+ static MinIOContainer minio =
+ new MinIOContainer("minio/minio:latest")
+ .withUserName(ACCESS_KEY)
+ .withPassword(SECRET_KEY);
+
+ private static S3Client adminClient;
+ private static int bucketCounter;
+
+ private String bucket;
+ private S3OutputSink sink;
+ private S3InputSource source;
+ private InProcessProcessedLedger ledger;
+
+ @BeforeEach
+ void setUp() {
+ if (adminClient == null) {
+ adminClient =
+ S3Client.builder()
+ .endpointOverride(URI.create(minio.getS3URL()))
+ .httpClient(UrlConnectionHttpClient.create())
+ .region(Region.US_EAST_1)
+ .credentialsProvider(
+ StaticCredentialsProvider.create(
+ AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
+ .serviceConfiguration(
+ S3Configuration.builder().pathStyleAccessEnabled(true).build())
+ .build();
+ }
+ bucket = "policy-outbox-" + ++bucketCounter;
+ adminClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
+
+ ApplicationProperties properties = new ApplicationProperties();
+ properties.getPolicies().setAllowPrivateS3Endpoints(true);
+ S3ConnectionPool pool = new S3ConnectionPool(properties);
+ ledger = new InProcessProcessedLedger();
+ sink = new S3OutputSink(pool, ledger);
+ source = new S3InputSource(pool);
+ }
+
+ @Test
+ void uploadsOutputsUnderThePrefix() throws IOException {
+ List results =
+ sink.deliver(
+ new OutputDelivery("run-1", POLICY),
+ List.of(output("doc.pdf", "pdf bytes")),
+ outputSpec("processed/"));
+
+ assertThat(results).hasSize(1);
+ assertThat(results.get(0).getFileName()).isEqualTo("s3://" + bucket + "/processed/doc.pdf");
+ assertThat(objectContent("processed/doc.pdf")).isEqualTo("pdf bytes");
+ }
+
+ @Test
+ void anExistingKeyIsNeverOverwritten() throws IOException {
+ adminClient.putObject(
+ PutObjectRequest.builder().bucket(bucket).key("doc.pdf").build(),
+ RequestBody.fromString("theirs", StandardCharsets.UTF_8));
+
+ List results =
+ sink.deliver(
+ new OutputDelivery("run-1", POLICY),
+ List.of(output("doc.pdf", "ours")),
+ outputSpec(""));
+
+ assertThat(results.get(0).getFileName()).isEqualTo("s3://" + bucket + "/doc (1).pdf");
+ assertThat(objectContent("doc.pdf")).isEqualTo("theirs");
+ assertThat(objectContent("doc (1).pdf")).isEqualTo("ours");
+ }
+
+ @Test
+ void aPolicyWritingIntoItsWatchedBucketSkipsItsOwnOutputsButAnotherPolicyChains()
+ throws IOException {
+ sink.deliver(
+ new OutputDelivery("run-1", POLICY),
+ List.of(output("result.pdf", "produced")),
+ outputSpec(""));
+
+ // The producing policy's sweep sees its own output at the recorded gate and skips it.
+ assertThat(source.resolve(inputSpec(), new RecordingContext(POLICY))).isEmpty();
+
+ // A different policy watching the same bucket has no row and processes it - chaining.
+ List chained = source.resolve(inputSpec(), new RecordingContext("p2"));
+ assertThat(chained).hasSize(1);
+ try (InputStream stream = chained.get(0).inputs().primary().get(0).getInputStream()) {
+ assertThat(new String(stream.readAllBytes(), StandardCharsets.UTF_8))
+ .isEqualTo("produced");
+ }
+ }
+
+ private OutputSpec outputSpec(String prefix) {
+ return new OutputSpec(
+ "s3",
+ Map.of(
+ "bucket", bucket,
+ "prefix", prefix,
+ "endpoint", minio.getS3URL(),
+ "accessKeyId", ACCESS_KEY,
+ "secretAccessKey", SECRET_KEY));
+ }
+
+ private InputSpec inputSpec() {
+ return new InputSpec(
+ "s3",
+ Map.of(
+ "bucket", bucket,
+ "endpoint", minio.getS3URL(),
+ "accessKeyId", ACCESS_KEY,
+ "secretAccessKey", SECRET_KEY));
+ }
+
+ private String objectContent(String key) throws IOException {
+ try (ResponseInputStream stream =
+ adminClient.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build())) {
+ return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private static Resource output(String name, String content) {
+ return new ByteArrayResource(content.getBytes(StandardCharsets.UTF_8)) {
+ @Override
+ public String getFilename() {
+ return name;
+ }
+ };
+ }
+
+ private class RecordingContext implements ResolveContext {
+
+ private final String policyId;
+
+ private RecordingContext(String policyId) {
+ this.policyId = policyId;
+ }
+
+ @Override
+ public boolean claim(String identity, String gate, Supplier contentHash) {
+ return ledger.claim(policyId, identity, gate, contentHash);
+ }
+
+ @Override
+ public void settle(
+ String identity, String finalGate, String finalContentHash, boolean success) {
+ ledger.settle(policyId, identity, finalGate, finalContentHash, success);
+ }
+
+ @Override
+ public boolean allSettledDone(String identity) {
+ return ledger.allSettledDone(identity);
+ }
+
+ @Override
+ public void reportPresent(Collection identities) {}
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkTest.java
new file mode 100644
index 0000000000..8240bcc4e1
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkTest.java
@@ -0,0 +1,267 @@
+package stirling.software.proprietary.policy.output;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.Resource;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.job.ResultFile;
+import stirling.software.proprietary.policy.ledger.ClaimState;
+import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
+import stirling.software.proprietary.policy.ledger.ProcessedFileStatus;
+import stirling.software.proprietary.policy.model.OutputSpec;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+/**
+ * Tests for {@link S3OutputSink}: the ledger row exists before the object is visible, collisions
+ * re-pick names, ad-hoc runs record nothing, and encrypted-bucket ETags are re-recorded.
+ */
+@ExtendWith(MockitoExtension.class)
+class S3OutputSinkTest {
+
+ private static final String POLICY = "p1";
+ private static final String BUCKET = "outbox-bucket";
+ private static final OutputDelivery DELIVERY = new OutputDelivery("run-1", POLICY);
+ private static final OutputDelivery AD_HOC = new OutputDelivery("run-2", null);
+
+ @Mock private S3Client s3Client;
+
+ private S3OutputSink sink;
+ private InProcessProcessedLedger ledger;
+ private final List puts = new ArrayList<>();
+
+ @BeforeEach
+ void setUp() {
+ ledger = new InProcessProcessedLedger();
+ sink =
+ new S3OutputSink(
+ new S3ConnectionPool(new ApplicationProperties(), config -> s3Client),
+ ledger);
+ }
+
+ @Test
+ void recordsTheRowBeforeTheObjectBecomesVisible() throws IOException {
+ // The row for the exact key must already be settled DONE at the moment the PUT runs -
+ // record-before-visible, asserted from inside the upload itself.
+ List stateAtPutTime = new ArrayList<>();
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenAnswer(
+ invocation -> {
+ PutObjectRequest request = invocation.getArgument(0);
+ puts.add(request);
+ stateAtPutTime.add(stateFor(identity(request.key())));
+ return PutObjectResponse.builder().eTag(quotedMd5("data")).build();
+ });
+
+ List results =
+ sink.deliver(DELIVERY, List.of(output("doc.pdf", "data")), spec());
+
+ assertEquals(1, results.size());
+ assertEquals("s3://" + BUCKET + "/processed/doc.pdf", results.get(0).getFileName());
+ assertEquals(4, results.get(0).getFileSize());
+ assertNotNull(stateAtPutTime.get(0));
+ assertEquals(ProcessedFileStatus.DONE, stateAtPutTime.get(0).status());
+ assertEquals(md5("data"), stateAtPutTime.get(0).gate());
+ assertTrue(puts.get(0).ifNoneMatch() != null);
+ }
+
+ @Test
+ void aTakenKeyIsForgottenAndRePicked() throws IOException {
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenAnswer(
+ invocation -> {
+ PutObjectRequest request = invocation.getArgument(0);
+ puts.add(request);
+ if (puts.size() == 1) {
+ throw s3Error(412, "PreconditionFailed");
+ }
+ return PutObjectResponse.builder().eTag(quotedMd5("data")).build();
+ });
+
+ List results =
+ sink.deliver(DELIVERY, List.of(output("doc.pdf", "data")), spec());
+
+ assertEquals("s3://" + BUCKET + "/processed/doc (1).pdf", results.get(0).getFileName());
+ // The lost candidate's row is gone; only the delivered key is recorded.
+ assertNull(stateFor(identity("processed/doc.pdf")));
+ assertNotNull(stateFor(identity("processed/doc (1).pdf")));
+ }
+
+ @Test
+ void anEncryptedBucketETagIsReRecordedAtTheActualGate() throws IOException {
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenReturn(PutObjectResponse.builder().eTag("\"kms-opaque-etag\"").build());
+
+ sink.deliver(DELIVERY, List.of(output("doc.pdf", "data")), spec());
+
+ assertEquals("kms-opaque-etag", stateFor(identity("processed/doc.pdf")).gate());
+ }
+
+ @Test
+ void anAdHocDeliveryRecordsNothing() throws IOException {
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenReturn(PutObjectResponse.builder().eTag(quotedMd5("data")).build());
+
+ sink.deliver(AD_HOC, List.of(output("doc.pdf", "data")), spec());
+
+ assertNull(stateFor(identity("processed/doc.pdf")));
+ }
+
+ @Test
+ void aFailedUploadForgetsItsRowAndThrows() {
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenThrow(SdkClientException.create("connection refused"));
+
+ assertThrows(
+ IOException.class,
+ () -> sink.deliver(DELIVERY, List.of(output("doc.pdf", "data")), spec()));
+
+ assertNull(stateFor(identity("processed/doc.pdf")));
+ }
+
+ @Test
+ void aStoreWithoutConditionalPutsFallsBackToExistenceChecks() throws IOException {
+ when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(s3Error(404, "NotFound"));
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenAnswer(
+ invocation -> {
+ PutObjectRequest request = invocation.getArgument(0);
+ puts.add(request);
+ if (request.ifNoneMatch() != null) {
+ throw s3Error(501, "NotImplemented");
+ }
+ return PutObjectResponse.builder().eTag(quotedMd5("data")).build();
+ });
+
+ List results =
+ sink.deliver(DELIVERY, List.of(output("doc.pdf", "data")), spec());
+
+ // Same key, second attempt unconditional.
+ assertEquals("s3://" + BUCKET + "/processed/doc.pdf", results.get(0).getFileName());
+ assertEquals(2, puts.size());
+ assertNull(puts.get(1).ifNoneMatch());
+ assertNotNull(stateFor(identity("processed/doc.pdf")));
+ }
+
+ @Test
+ void aBarePrefixGetsItsSlash() throws IOException {
+ when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class)))
+ .thenAnswer(
+ invocation -> {
+ puts.add(invocation.getArgument(0));
+ return PutObjectResponse.builder().eTag(quotedMd5("data")).build();
+ });
+
+ sink.deliver(DELIVERY, List.of(output("doc.pdf", "data")), spec("processed"));
+
+ assertEquals("processed/doc.pdf", puts.get(0).key());
+ }
+
+ @Test
+ void validateRejectsBadConfigShape() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> sink.validate(new OutputSpec("s3", Map.of())));
+ // Credentials are required, never the server's own identity.
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> sink.validate(new OutputSpec("s3", Map.of("bucket", BUCKET))));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ sink.validate(
+ new OutputSpec(
+ "s3", Map.of("bucket", BUCKET, "accessKeyId", "AKIA"))));
+ }
+
+ @Test
+ void supportsOnlyS3Specs() {
+ assertTrue(sink.supports(spec()));
+ assertFalse(sink.supports(OutputSpec.inline()));
+ assertFalse(sink.supports(null));
+ }
+
+ private static OutputSpec spec() {
+ return spec("processed/");
+ }
+
+ private static OutputSpec spec(String prefix) {
+ return new OutputSpec(
+ "s3",
+ Map.of(
+ "bucket",
+ BUCKET,
+ "prefix",
+ prefix,
+ "accessKeyId",
+ "AKIAEXAMPLE",
+ "secretAccessKey",
+ "shh"));
+ }
+
+ private static String identity(String key) {
+ return "s3://" + BUCKET + "/" + key;
+ }
+
+ private ClaimState stateFor(String identity) {
+ return ledger.statesFor(POLICY, List.of(identity)).get(identity);
+ }
+
+ private static Resource output(String name, String content) {
+ return new ByteArrayResource(content.getBytes(StandardCharsets.UTF_8)) {
+ @Override
+ public String getFilename() {
+ return name;
+ }
+ };
+ }
+
+ private static String md5(String content) {
+ try {
+ return HexFormat.of()
+ .formatHex(
+ MessageDigest.getInstance("MD5")
+ .digest(content.getBytes(StandardCharsets.UTF_8)));
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static String quotedMd5(String content) {
+ return "\"" + md5(content) + "\"";
+ }
+
+ private static AwsServiceException s3Error(int status, String code) {
+ return S3Exception.builder().statusCode(status).message(code).build();
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java
index d1d62b7dbe..295b4785b8 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java
@@ -28,6 +28,7 @@ import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
+import stirling.software.proprietary.util.SecretMasker;
/**
* Tests for {@link SourceController}'s delete guard: a source still referenced by a policy is
@@ -109,6 +110,107 @@ class SourceControllerTest {
assertEquals(404, controller.delete("nope").getStatusCode().value());
}
+ @Test
+ void documentCountsForTheEditorReturnsTheTeamSeries() {
+ ResponseEntity> response = controller.documentCounts(EditorSource.ID);
+
+ assertEquals(200, response.getStatusCode().value());
+ assertEquals(30, response.getBody().size());
+ }
+
+ @Test
+ void theEditorIsBuiltInAndCannotBeDeleted() {
+ ResponseStatusException ex =
+ assertThrows(
+ ResponseStatusException.class, () -> controller.delete(EditorSource.ID));
+
+ assertEquals(400, ex.getStatusCode().value());
+ }
+
+ @Test
+ void theEditorIsBuiltInAndCannotBeSaved() {
+ Source editor = new Source(null, "Editor", "editor", Map.of(), true, null, null);
+
+ ResponseStatusException ex =
+ assertThrows(ResponseStatusException.class, () -> controller.save(editor));
+
+ assertEquals(400, ex.getStatusCode().value());
+ }
+
+ @Test
+ void readsReturnSecretsAsTheRedactionSentinel() {
+ Source saved = sourceStore.save(s3Source("shh"));
+
+ Source read = controller.get(saved.id()).getBody();
+
+ assertEquals(SecretMasker.REDACTED, read.options().get("secretAccessKey"));
+ assertEquals("AKIAEXAMPLE", read.options().get("accessKeyId"));
+ // The store itself keeps the real value.
+ assertEquals(
+ "shh", sourceStore.get(saved.id()).orElseThrow().options().get("secretAccessKey"));
+ }
+
+ @Test
+ void savingTheSentinelBackKeepsTheStoredSecret() {
+ Source saved = sourceStore.save(s3Source("shh"));
+
+ Source edited =
+ new Source(
+ saved.id(),
+ "Renamed",
+ saved.type(),
+ Map.of(
+ "bucket", "inbox",
+ "accessKeyId", "AKIAEXAMPLE",
+ "secretAccessKey", SecretMasker.REDACTED),
+ true,
+ saved.owner(),
+ saved.teamId());
+ Source response = controller.save(edited).getBody();
+
+ assertEquals(
+ "shh", sourceStore.get(saved.id()).orElseThrow().options().get("secretAccessKey"));
+ // The save response is masked too; only the store sees the real value.
+ assertEquals(SecretMasker.REDACTED, response.options().get("secretAccessKey"));
+ }
+
+ @Test
+ void savingANewSecretReplacesTheStoredOne() {
+ Source saved = sourceStore.save(s3Source("old-secret"));
+
+ Source edited =
+ new Source(
+ saved.id(),
+ saved.name(),
+ saved.type(),
+ Map.of(
+ "bucket", "inbox",
+ "accessKeyId", "AKIAEXAMPLE",
+ "secretAccessKey", "new-secret"),
+ true,
+ saved.owner(),
+ saved.teamId());
+ controller.save(edited);
+
+ assertEquals(
+ "new-secret",
+ sourceStore.get(saved.id()).orElseThrow().options().get("secretAccessKey"));
+ }
+
+ private static Source s3Source(String secret) {
+ return new Source(
+ null,
+ "Bucket intake",
+ "s3",
+ Map.of(
+ "bucket", "inbox",
+ "accessKeyId", "AKIAEXAMPLE",
+ "secretAccessKey", secret),
+ true,
+ "owner",
+ null);
+ }
+
private static Source folderSource() {
return new Source(
null, "Claims intake", "folder", Map.of("directory", "/in"), true, "owner", null);
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java
index 275f076db3..3f63e18407 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java
@@ -56,9 +56,10 @@ class SourceOverviewServiceTest {
SourcesResponse response = service.overview();
- assertEquals(3, response.sources().size());
- // Sorted most-referenced first, so the shared source A leads.
- assertEquals(a.id(), response.sources().get(0).id());
+ assertEquals(4, response.sources().size());
+ // The built-in editor is pinned first; persisted sources follow, most-referenced leading.
+ assertEquals(EditorSource.ID, response.sources().get(0).id());
+ assertEquals(a.id(), response.sources().get(1).id());
SourceView av = find(response, a.id());
assertEquals(2, av.referenceCount());
@@ -123,13 +124,57 @@ class SourceOverviewServiceTest {
SourcesResponse response = scoped.overview();
- assertEquals(1, response.sources().size());
- SourceView view = response.sources().get(0);
- assertEquals(ours.id(), view.id());
+ assertEquals(2, response.sources().size());
+ assertEquals(EditorSource.ID, response.sources().get(0).id());
+ SourceView view = find(response, ours.id());
assertEquals(1, view.referenceCount());
assertEquals(List.of(1L, 1L, 0L), response.kpis().stream().map(SourceKpi::value).toList());
}
+ @Test
+ void theEditorSourceIsAlwaysPresentEvenWithNoConnections() {
+ SourcesResponse response = service.overview();
+
+ assertEquals(1, response.sources().size());
+ SourceView editor = response.sources().get(0);
+ assertEquals(EditorSource.ID, editor.id());
+ assertEquals("editor", editor.type());
+ assertEquals("active", editor.status());
+ assertEquals(0, editor.referenceCount());
+ // KPIs describe configured connections, so the built-in editor is left out of them.
+ assertEquals(List.of(0L, 0L, 0L), response.kpis().stream().map(SourceKpi::value).toList());
+ }
+
+ @Test
+ void theEditorSourceIsUsedByEveryPolicyThatRunsFromIt() {
+ editorPolicy("Redact on upload");
+ editorPolicy("Classify on upload");
+ // A folder-sourced policy does not target the editor, so it must not inflate the count.
+ policyReferencing("Folder sweep", source("Folder", "/f").id());
+
+ SourceView editor = find(service.overview(), EditorSource.ID);
+
+ assertEquals(2, editor.referenceCount());
+ assertTrue(
+ editor.referencingPolicies().stream()
+ .map(SourceView.PolicyRef::name)
+ .toList()
+ .containsAll(List.of("Redact on upload", "Classify on upload")));
+ }
+
+ @Test
+ void theEditorSourceReportsTheTeamsRecordedDocumentThroughput() {
+ // Login disabled, so the team is null and the editor shares the global counter bucket.
+ docCounter.record(EditorSource.counterKey(null), 4);
+ docCounter.record(EditorSource.counterKey(null), 6);
+
+ SourceView editor = find(service.overview(), EditorSource.ID);
+
+ assertEquals(10, editor.docsTotal());
+ assertEquals(10, editor.docs24h());
+ assertEquals(10, editor.docs30d());
+ }
+
@Test
void documentCountsReflectRecordedDocs() {
Source a = source("A", "/a");
@@ -177,6 +222,22 @@ class SourceOverviewServiceTest {
OutputSpec.inline()));
}
+ /**
+ * A policy that targets the editor: membership rides in its output metadata, not a sourceId.
+ */
+ private void editorPolicy(String name) {
+ policyStore.save(
+ new Policy(
+ null,
+ name,
+ "owner",
+ true,
+ null,
+ List.of(),
+ List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
+ new OutputSpec("inline", Map.of("sources", List.of("editor")))));
+ }
+
private void teamPolicy(String name, Long teamId, String... sourceIds) {
policyStore.save(
new Policy(
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java
index 889c50ee44..3f81392285 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java
@@ -70,6 +70,41 @@ class ScheduleTriggerTest {
verify(policyRunner, times(1)).run(eq(policy));
}
+ @Test
+ void anIntervalMatchingTheSweepPeriodFiresEverySweepDespiteJitter() {
+ Policy policy = scheduled("p1", new Schedule.Every(1, Schedule.Unit.MINUTES));
+ when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
+
+ Instant t0 = Instant.parse("2026-06-05T10:00:00Z");
+ trigger.sweep(t0); // baseline
+ // The sweep that fires runs a few ms late (scheduler jitter)...
+ trigger.sweep(t0.plusSeconds(60).plusMillis(5));
+ verify(policyRunner, times(1)).run(eq(policy));
+
+ // ...and the next sweep lands exactly on the 60s grid. Anchoring lastFired to the due
+ // time (not the jittered observation) means this must still fire, not alias to skip.
+ trigger.sweep(t0.plusSeconds(120));
+ verify(policyRunner, times(2)).run(eq(policy));
+ }
+
+ @Test
+ void aGapFiresOnceNotOncePerMissedInterval() {
+ Policy policy = scheduled("p1", new Schedule.Every(1, Schedule.Unit.MINUTES));
+ when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
+
+ Instant t0 = Instant.parse("2026-06-05T10:00:00Z");
+ trigger.sweep(t0); // baseline
+ // Ten minutes of downtime: nine missed due points collapse into one firing.
+ trigger.sweep(t0.plusSeconds(600));
+ verify(policyRunner, times(1)).run(eq(policy));
+
+ // Not due again until a full interval after the latest due point.
+ trigger.sweep(t0.plusSeconds(630));
+ verify(policyRunner, times(1)).run(eq(policy));
+ trigger.sweep(t0.plusSeconds(660));
+ verify(policyRunner, times(2)).run(eq(policy));
+ }
+
@Test
void doesNotFireBeforeTheNextScheduledTime() {
Policy policy = scheduled("p1", new Schedule.Daily(LocalTime.of(3, 0))); // 03:00 UTC daily
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AuditServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AuditServiceTest.java
index a90fde6b85..c2d14d1307 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AuditServiceTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AuditServiceTest.java
@@ -36,6 +36,7 @@ import jakarta.servlet.http.HttpServletResponse;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.service.InternalApiClient;
+import stirling.software.proprietary.audit.AuditContext;
import stirling.software.proprietary.audit.AuditEventType;
import stirling.software.proprietary.audit.AuditLevel;
import stirling.software.proprietary.audit.Audited;
@@ -718,6 +719,57 @@ class AuditServiceTest {
}
}
+ @Nested
+ @DisplayName("addAutomationContext")
+ class AddAutomationContext {
+
+ @Test
+ @DisplayName("caps and strips newlines from a spoofable policy-name header")
+ void capsPolicyNameHeader() {
+ MockHttpServletRequest req =
+ new MockHttpServletRequest("POST", "/api/v1/misc/compress-pdf");
+ req.addHeader(InternalApiClient.POLICY_NAME_HEADER, "evil\r\nname" + "x".repeat(500));
+ Map data = new HashMap<>();
+
+ service.addAutomationContext(data, req);
+
+ String stored = (String) data.get("policyName");
+ assertThat(stored).hasSize(200);
+ assertThat(stored).doesNotContain("\r").doesNotContain("\n");
+ }
+
+ @Test
+ @DisplayName("caps the number of persisted policy steps")
+ void capsPolicySteps() {
+ MockHttpServletRequest req = new MockHttpServletRequest("POST", "/api/v1/policies/run");
+ req.setAttribute(
+ AuditContext.REQ_ATTR_POLICY_STEPS,
+ java.util.stream.IntStream.range(0, 100)
+ .mapToObj(i -> "/api/v1/misc/step-" + i)
+ .toList());
+ Map data = new HashMap<>();
+
+ service.addAutomationContext(data, req);
+
+ @SuppressWarnings("unchecked")
+ java.util.List stored = (java.util.List) data.get("policySteps");
+ assertThat(stored).hasSize(50);
+ }
+
+ @Test
+ @DisplayName("marks automation from the header")
+ void marksAutomationHeader() {
+ MockHttpServletRequest req =
+ new MockHttpServletRequest("POST", "/api/v1/misc/compress-pdf");
+ req.addHeader(InternalApiClient.AUTOMATION_HEADER, "true");
+ Map data = new HashMap<>();
+
+ service.addAutomationContext(data, req);
+
+ assertThat(data).containsEntry("automation", Boolean.TRUE);
+ }
+ }
+
// ===== helpers =====
private ProceedingJoinPoint joinPoint() {
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalAuditReadServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalAuditReadServiceTest.java
new file mode 100644
index 0000000000..b357fbc2e3
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalAuditReadServiceTest.java
@@ -0,0 +1,73 @@
+package stirling.software.proprietary.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
+
+import stirling.software.proprietary.model.security.PersistentAuditEvent;
+import stirling.software.proprietary.repository.PersistentAuditEventRepository;
+
+/**
+ * The scan must drop read/polling noise (UI_DATA/HTTP_REQUEST) at the query level. Otherwise a busy
+ * scope's recent rows fill with noise and the visible audit list shrinks as traffic grows.
+ */
+@ExtendWith(MockitoExtension.class)
+class PortalAuditReadServiceTest {
+
+ @Mock private PersistentAuditEventRepository repo;
+
+ @InjectMocks private PortalAuditReadService service;
+
+ private static Page emptyPage() {
+ return new PageImpl<>(List.of());
+ }
+
+ @Test
+ void serverScanExcludesReadNoiseAndNeverPullsEverything() {
+ when(repo.findByTypeNotIn(anyList(), any(Pageable.class))).thenReturn(emptyPage());
+
+ service.serverEvents();
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor> excluded = ArgumentCaptor.forClass(List.class);
+ verify(repo).findByTypeNotIn(excluded.capture(), any(Pageable.class));
+ assertThat(excluded.getValue()).contains("UI_DATA", "HTTP_REQUEST");
+ verify(repo, never()).findAll(any(Pageable.class));
+ }
+
+ @Test
+ void teamScanExcludesReadNoiseForTheTeamPrincipals() {
+ when(repo.findByTypeNotInAndPrincipalIn(anyList(), anyList(), any(Pageable.class)))
+ .thenReturn(emptyPage());
+
+ service.scopedEvents("team:1", List.of("a@acme.com", "b@acme.com"));
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor> excluded = ArgumentCaptor.forClass(List.class);
+ verify(repo)
+ .findByTypeNotInAndPrincipalIn(excluded.capture(), anyList(), any(Pageable.class));
+ assertThat(excluded.getValue()).contains("UI_DATA", "HTTP_REQUEST");
+ }
+
+ @Test
+ void emptyTeamPrincipalsShortCircuitToNoQuery() {
+ assertThat(service.scopedEvents("team:1", List.of())).isEmpty();
+ verify(repo, never())
+ .findByTypeNotInAndPrincipalIn(anyList(), anyList(), any(Pageable.class));
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalDocumentsServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalDocumentsServiceTest.java
new file mode 100644
index 0000000000..1afc0af4a4
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalDocumentsServiceTest.java
@@ -0,0 +1,73 @@
+package stirling.software.proprietary.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import java.time.Instant;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import stirling.software.proprietary.audit.AuditEventType;
+import stirling.software.proprietary.audit.PortalAuditEventRow;
+import stirling.software.proprietary.model.api.documents.PortalReviewDocumentDto;
+
+import tools.jackson.databind.json.JsonMapper;
+
+/**
+ * The Documents feed showed a policy run's internal steps as ordinary "API" traffic. They should
+ * read as automation so a policy run isn't confused with a customer's own API integration.
+ */
+@ExtendWith(MockitoExtension.class)
+class PortalDocumentsServiceTest {
+
+ @Mock private PortalAuditReadService auditReadService;
+
+ private PortalDocumentsService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new PortalDocumentsService(auditReadService, JsonMapper.builder().build());
+ }
+
+ private static PortalAuditEventRow row(long id, String data) {
+ return new PortalAuditEventRow(
+ id, "con.yoh13@gmail.com", AuditEventType.PDF_PROCESS.name(), data, Instant.now());
+ }
+
+ private PortalReviewDocumentDto onlyDoc(String data) {
+ when(auditReadService.serverEvents()).thenReturn(List.of(row(1L, data)));
+ return service.serverDocuments().getDocuments().get(0);
+ }
+
+ @Test
+ void policyStepDocumentIsLabelledAutomation() {
+ PortalReviewDocumentDto doc =
+ onlyDoc(
+ "{\"path\":\"/api/v1/security/auto-redact\",\"automation\":true,"
+ + "\"__origin\":\"API\",\"files\":[{\"name\":\"mushroom life.pdf\","
+ + "\"type\":\"application/pdf\"}],\"statusCode\":200}");
+
+ assertThat(doc.getName()).isEqualTo("mushroom life.pdf");
+ assertThat(doc.getAction()).isEqualTo("Auto Redact");
+ // Was product "API" (loopback used the API key); now clearly a policy automation step.
+ assertThat(doc.getProduct()).isEqualTo("Automation");
+ assertThat(doc.getSource()).isEqualTo("Policy automation");
+ }
+
+ @Test
+ void directApiDocumentStaysApi() {
+ PortalReviewDocumentDto doc =
+ onlyDoc(
+ "{\"path\":\"/api/v1/misc/compress-pdf\",\"__origin\":\"API\","
+ + "\"files\":[{\"name\":\"a.pdf\",\"type\":\"application/pdf\"}],"
+ + "\"statusCode\":200}");
+
+ assertThat(doc.getProduct()).isEqualTo("API");
+ assertThat(doc.getSource()).isEqualTo("API integration");
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalInfraAuditServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalInfraAuditServiceTest.java
new file mode 100644
index 0000000000..8d225367f3
--- /dev/null
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/PortalInfraAuditServiceTest.java
@@ -0,0 +1,149 @@
+package stirling.software.proprietary.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import java.time.Instant;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import stirling.software.proprietary.audit.AuditEventType;
+import stirling.software.proprietary.audit.PortalAuditEventRow;
+import stirling.software.proprietary.model.api.audit.InfraAuditEventDto;
+
+import tools.jackson.databind.json.JsonMapper;
+
+/**
+ * The Infrastructure → Audit tab reproduces the reported bug: a policy run showed as a bare "Run"
+ * row plus a separate "Auto Redact" row with no hint either belonged to a policy.
+ */
+@ExtendWith(MockitoExtension.class)
+class PortalInfraAuditServiceTest {
+
+ @Mock private PortalAuditReadService auditReadService;
+
+ private PortalInfraAuditService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new PortalInfraAuditService(auditReadService, JsonMapper.builder().build());
+ }
+
+ private static PortalAuditEventRow row(long id, String data) {
+ return new PortalAuditEventRow(
+ id,
+ "con.yoh13@gmail.com",
+ AuditEventType.PDF_PROCESS.name(),
+ data,
+ Instant.parse("2026-07-09T10:18:24Z"));
+ }
+
+ private InfraAuditEventDto onlyEvent(String data) {
+ when(auditReadService.serverEvents()).thenReturn(List.of(row(1L, data)));
+ return service.serverAuditLog().getEvents().get(0);
+ }
+
+ @Test
+ void policyDispatchShowsPolicyNameAndTheToolsItRuns() {
+ String data =
+ "{\"path\":\"/api/v1/policies/run\",\"policyName\":\"Redaction\","
+ + "\"policySteps\":[\"/api/v1/security/auto-redact\","
+ + "\"/api/v1/misc/compress-pdf\"],\"statusCode\":202,"
+ + "\"latencyMs\":5}";
+ when(auditReadService.serverEvents()).thenReturn(List.of(row(1L, data)));
+
+ var resp = service.serverAuditLog();
+ InfraAuditEventDto e = resp.getEvents().get(0);
+
+ // Was "Run" / "Run" under "processing"; now names the policy, lists what it ran, and
+ // badges as its own "policy" category with its own summary count.
+ assertThat(e.getAction()).isEqualTo("Redaction");
+ assertThat(e.getCategory()).isEqualTo("policy");
+ assertThat(e.getTarget()).isEqualTo("Auto Redact, Compress PDF");
+ assertThat(resp.getSummary().getPolicy()).isEqualTo(1);
+ }
+
+ @Test
+ void internalPipelineStepIsFlaggedAsAutomation() {
+ InfraAuditEventDto e =
+ onlyEvent(
+ "{\"path\":\"/api/v1/security/auto-redact\",\"automation\":true,"
+ + "\"files\":[{\"name\":\"mushroom life.pdf\"}],"
+ + "\"statusCode\":200,\"latencyMs\":300}");
+
+ // Was a bare "Auto Redact" indistinguishable from a direct user action.
+ assertThat(e.getAction()).isEqualTo("Auto Redact (automation)");
+ assertThat(e.getCategory()).isEqualTo("security");
+ assertThat(e.getTarget()).isEqualTo("mushroom life.pdf");
+ }
+
+ @Test
+ void internalStepCarryingItsPolicyNameLinksBackToThePolicy() {
+ InfraAuditEventDto e =
+ onlyEvent(
+ "{\"path\":\"/api/v1/security/auto-redact\",\"automation\":true,"
+ + "\"policyName\":\"Redaction demo\",\"files\":[{\"name\":"
+ + "\"demo.pdf\"}],\"statusCode\":200}");
+
+ // The forwarded policy name makes the step's origin unmistakable.
+ assertThat(e.getAction()).isEqualTo("Auto Redact (policy: Redaction demo)");
+ assertThat(e.getCategory()).isEqualTo("security");
+ assertThat(e.getTarget()).isEqualTo("demo.pdf");
+ }
+
+ @Test
+ void adHocRunWithoutNameStillReadsAsPolicyRun() {
+ InfraAuditEventDto e =
+ onlyEvent(
+ "{\"path\":\"/api/v1/policies/run/stream\",\"statusCode\":200,"
+ + "\"latencyMs\":4}");
+
+ assertThat(e.getAction()).isEqualTo("Policy run");
+ assertThat(e.getTarget()).isEqualTo("Pipeline");
+ }
+
+ @Test
+ void directToolRunIsUnchanged() {
+ InfraAuditEventDto e =
+ onlyEvent(
+ "{\"path\":\"/api/v1/misc/compress-pdf\","
+ + "\"files\":[{\"name\":\"a.pdf\"}],\"statusCode\":200,"
+ + "\"latencyMs\":100}");
+
+ assertThat(e.getAction()).isEqualTo("Compress PDF");
+ assertThat(e.getCategory()).isEqualTo("processing");
+ assertThat(e.getTarget()).isEqualTo("a.pdf");
+ }
+
+ /**
+ * A spoofed X-Stirling-Policy-Name header lands in audit data as policyName on a direct tool
+ * call (no automation marker, non-run path). It must NOT flip the row into a "policy" dispatch
+ * that overwrites the real action and hides the affected file - only a real /policies/.../run
+ * URI does that.
+ */
+ @Test
+ void forgedPolicyNameOnDirectCallCannotMaskTheRealAction() {
+ when(auditReadService.serverEvents())
+ .thenReturn(
+ List.of(
+ row(
+ 1L,
+ "{\"path\":\"/api/v1/security/remove-password\","
+ + "\"policyName\":\"Daily cleanup\",\"files\":"
+ + "[{\"name\":\"secret.pdf\"}],\"statusCode\":200}")));
+
+ var resp = service.serverAuditLog();
+ InfraAuditEventDto e = resp.getEvents().get(0);
+
+ // Real op and file stay visible; the forged name does not become the action or category.
+ assertThat(e.getAction()).isEqualTo("Remove Password");
+ assertThat(e.getCategory()).isEqualTo("security");
+ assertThat(e.getTarget()).isEqualTo("secret.pdf");
+ assertThat(resp.getSummary().getPolicy()).isEqualTo(0);
+ }
+}
diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/util/SecretMaskerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/util/SecretMaskerTest.java
index d91aec2029..415117a446 100644
--- a/app/proprietary/src/test/java/stirling/software/proprietary/util/SecretMaskerTest.java
+++ b/app/proprietary/src/test/java/stirling/software/proprietary/util/SecretMaskerTest.java
@@ -14,8 +14,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link SecretMasker}.
*
*
Assumptions: - Key matching is case-insensitive via the pattern in SENSITIVE. - If the key
- * matches a sensitive pattern, the value is replaced with "***REDACTED***". - Nested maps and lists
- * are searched recursively. - Null maps and null values are ignored or returned as null. -
+ * matches a sensitive pattern, the value is replaced with SecretMasker.REDACTED. - Nested maps and
+ * lists are searched recursively. - Null maps and null values are ignored or returned as null. -
* Non-sensitive keys/values remain unchanged.
*/
class SecretMaskerTest {
@@ -40,7 +40,7 @@ class SecretMaskerTest {
Map result = SecretMasker.mask(input);
- assertEquals("***REDACTED***", result.get("password"));
+ assertEquals(SecretMasker.REDACTED, result.get("password"));
assertEquals("john", result.get("username"));
}
@@ -55,11 +55,54 @@ class SecretMaskerTest {
Map result = SecretMasker.mask(input);
- assertEquals("***REDACTED***", result.get("Api-Key"));
- assertEquals("***REDACTED***", result.get("TOKEN"));
+ assertEquals(SecretMasker.REDACTED, result.get("Api-Key"));
+ assertEquals(SecretMasker.REDACTED, result.get("TOKEN"));
assertEquals("keepme", result.get("normal"));
}
+ @Test
+ @DisplayName("restoreRedacted swaps sentinels for stored values, leaves the rest")
+ void restoreRedactedRoundTripsAnEdit() {
+ Map stored =
+ Map.of("secretAccessKey", "shh", "accessKeyId", "AKIAEXAMPLE");
+ Map incoming =
+ Map.of(
+ "secretAccessKey", SecretMasker.REDACTED,
+ "accessKeyId", "AKIA-NEW",
+ "bucket", "inbox");
+
+ Map merged = SecretMasker.restoreRedacted(incoming, stored);
+
+ assertEquals("shh", merged.get("secretAccessKey"));
+ assertEquals("AKIA-NEW", merged.get("accessKeyId"));
+ assertEquals("inbox", merged.get("bucket"));
+ }
+
+ @Test
+ @DisplayName("restoreRedacted leaves a sentinel with no stored counterpart in place")
+ void restoreRedactedWithoutStoredValueStaysSentinel() {
+ Map merged =
+ SecretMasker.restoreRedacted(
+ Map.of("secretAccessKey", SecretMasker.REDACTED), Map.of());
+
+ assertEquals(SecretMasker.REDACTED, merged.get("secretAccessKey"));
+ }
+
+ @Test
+ @DisplayName("should mask camelCase secretAccessKey despite no word boundary")
+ void shouldMaskCamelCaseSecretAccessKey() {
+ Map input =
+ Map.of(
+ "secretAccessKey", "shh",
+ "accessKeyId", "AKIAEXAMPLE");
+
+ Map result = SecretMasker.mask(input);
+
+ assertEquals(SecretMasker.REDACTED, result.get("secretAccessKey"));
+ // Access key ids are username-like, not secrets.
+ assertEquals("AKIAEXAMPLE", result.get("accessKeyId"));
+ }
+
@Test
@DisplayName("should mask nested map sensitive keys")
void shouldMaskNestedMapSensitiveKeys() {
@@ -77,9 +120,9 @@ class SecretMaskerTest {
Map result = SecretMasker.mask(input);
Map outer = (Map) result.get("outer");
- assertEquals("***REDACTED***", outer.get("jwt"));
+ assertEquals(SecretMasker.REDACTED, outer.get("jwt"));
Map inner = (Map) outer.get("inner");
- assertEquals("***REDACTED***", inner.get("secret"));
+ assertEquals(SecretMasker.REDACTED, inner.get("secret"));
assertEquals("ok", inner.get("other"));
}
@@ -98,7 +141,7 @@ class SecretMaskerTest {
List> list = (List>) result.get("list");
Map first = (Map) list.get(0);
- assertEquals("***REDACTED***", first.get("token"));
+ assertEquals(SecretMasker.REDACTED, first.get("token"));
Map second = (Map) list.get(1);
assertEquals("john", second.get("username"));
assertEquals("stringValue", list.get(2));
@@ -170,7 +213,8 @@ class SecretMaskerTest {
Map outer = (Map) result.get("outer");
assertTrue(outer.containsKey(null), "Null key should be preserved");
assertEquals("plainText", outer.get(null), "Value for null key must not be masked");
- assertEquals("***REDACTED***", outer.get("password"), "Sensitive keys must be masked");
+ assertEquals(
+ SecretMasker.REDACTED, outer.get("password"), "Sensitive keys must be masked");
}
}
}
diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java
index 371060c032..28bbfbdea0 100644
--- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java
+++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java
@@ -117,7 +117,8 @@ public class AiCreateController {
user.getTeam().getId(),
source,
ProcessType.SINGLE_TOOL,
- BillingCategory.AI);
+ BillingCategory.AI,
+ null);
jobChargeService.chargeStandalone(ctx, 1);
} catch (RuntimeException e) {
log.warn(
diff --git a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java
index 90616bea55..18da81ae2b 100644
--- a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java
+++ b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java
@@ -173,7 +173,9 @@ public class PaygWalletController {
int spend = clampToInt(snap.periodSpendUnits());
Integer limit = snap.periodCapUnits() != null ? clampToInt(snap.periodCapUnits()) : null;
- CategoryBreakdown breakdown = buildBreakdown(teamId, snap.periodStart(), snap.periodEnd());
+ BreakdownPair breakdowns = buildBreakdowns(teamId, snap.periodStart(), snap.periodEnd());
+ UsageAnalytics analytics =
+ buildUsageAnalytics(teamId, snap.periodStart(), snap.periodEnd());
// Estimated bill = paid (Stripe-metered) docs this period × rate — the free portion was
// already netted out at charge time, so this is the metered total, not spend − grant.
@@ -203,30 +205,63 @@ public class PaygWalletController {
noCap,
billing.subscriptionId(),
spend,
- breakdown,
+ breakdowns.units(),
members,
- buildActivity(teamId));
+ buildActivity(teamId),
+ breakdowns.docs(),
+ analytics.docsProcessed(),
+ analytics.uniquePdfs(),
+ analytics.sizeMultiplierPdfs());
return ResponseEntity.ok(body);
}
- private CategoryBreakdown buildBreakdown(
+ /** Per-category size-scaled units + input-file counts for the same window. */
+ private record BreakdownPair(CategoryBreakdown units, CategoryBreakdown docs) {}
+
+ /** Period usage analytics: total input files, unique PDFs, and size-multiplier files. */
+ private record UsageAnalytics(int docsProcessed, int uniquePdfs, int sizeMultiplierPdfs) {}
+
+ private BreakdownPair buildBreakdowns(
Long teamId, LocalDateTime periodStart, LocalDateTime periodEnd) {
- Map byCategory = new HashMap<>();
+ Map units = new HashMap<>();
+ Map docs = new HashMap<>();
for (Object[] row :
- ledgerRepo.sumPeriodAmountByCategory(
+ ledgerRepo.sumPeriodByCategoryWithDocs(
teamId, LedgerEntryType.DEBIT, periodStart, periodEnd)) {
- if (row.length >= 2
- && row[0] instanceof BillingCategory cat
- && row[1] instanceof Number n) {
- byCategory.put(cat, n.longValue());
+ if (row.length >= 3 && row[0] instanceof BillingCategory cat) {
+ if (row[1] instanceof Number u) {
+ units.put(cat, u.longValue());
+ }
+ if (row[2] instanceof Number d) {
+ docs.put(cat, d.longValue());
+ }
}
}
+ return new BreakdownPair(categoryBreakdown(units), categoryBreakdown(docs));
+ }
+
+ private static CategoryBreakdown categoryBreakdown(Map byCategory) {
return new CategoryBreakdown(
clampToInt(byCategory.getOrDefault(BillingCategory.API, 0L)),
clampToInt(byCategory.getOrDefault(BillingCategory.AI, 0L)),
clampToInt(byCategory.getOrDefault(BillingCategory.AUTOMATION, 0L)));
}
+ private UsageAnalytics buildUsageAnalytics(
+ Long teamId, LocalDateTime periodStart, LocalDateTime periodEnd) {
+ List