Compare commits

...
Author SHA1 Message Date
Anthony Stirling 7f6bc5e956 Remove low-value narration comments 2026-07-13 20:33:56 +01:00
Anthony Stirling cb40d7778f Condense comments to single lines 2026-07-13 20:28:07 +01:00
Anthony Stirling 8cb03dba9e Align webhook source with S3 connections 2026-07-13 19:54:15 +01:00
Anthony Stirling 247b783c75 Merge remote-tracking branch 'origin/main' into claude/stirling-sources-setup-fe5517
# Conflicts:
#	frontend/editor/src/portal/components/sources/ConnectWizard.test.tsx
#	frontend/editor/src/portal/components/sources/ConnectWizard.tsx
#	frontend/editor/src/portal/components/sources/SourceDetailPanel.tsx
2026-07-13 16:43:36 +01:00
Anthony Stirling a9f7add87d Add webhook policy source 2026-07-13 16:40:48 +01:00
31 changed files with 2107 additions and 25 deletions
@@ -254,6 +254,9 @@ public class ApplicationProperties {
* in-network object store.
*/
private boolean allowPrivateS3Endpoints = false;
/** Max inbound webhook body (bytes); over it is rejected 413. Default 100 MB. */
private long webhookMaxBytes = 104857600L;
}
@Data
@@ -198,6 +198,8 @@ public class RequestUriUtils {
|| trimmedUri.startsWith("/readiness")
|| trimmedUri.startsWith(
"/api/v1/mobile-scanner/") // Mobile scanner endpoints (no auth)
// Webhook receiver: authenticated per-request by HMAC signature, not a session.
|| trimmedUri.startsWith("/api/v1/webhooks/")
|| trimmedUri.startsWith("/v1/api-docs")
// Workflow participant endpoints - access controlled by share tokens, not login
|| trimmedUri.startsWith("/api/v1/workflow/participant/")
@@ -176,6 +176,13 @@ class RequestUriUtilsTest {
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/api/v1/convert", ""));
}
@Test
void testIsPublicAuthEndpoint_webhookReceiver() {
// The webhook source receiver authenticates each delivery by HMAC signature, not a session.
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/api/v1/webhooks/whk_abc123", ""));
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/app/api/v1/webhooks/whk_abc123", "/app"));
}
@Test
void testIsPublicAuthEndpoint_withContextPath() {
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/app/login", "/app"));
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import stirling.software.proprietary.policy.model.InputSpec;
@@ -22,6 +23,12 @@ public interface InputSource {
/** Throws {@link IllegalArgumentException} on bad config. Called on save to fail fast. */
default void validate(InputSpec spec) {}
/** Normalise a source's options before persistence (default no-op); must not mutate the arg. */
default Map<String, Object> prepareOptionsForSave(
Map<String, Object> options, boolean isCreate) {
return options;
}
/**
* Resolve the spec into zero or more units of work, each carrying one run's files and a
* completion hook. Empty list means nothing to run right now. Discovery is read-only - files
@@ -0,0 +1,191 @@
package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.ledger.FolderIdentities;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.webhook.WebhookConfig;
import stirling.software.proprietary.policy.webhook.WebhookIds;
import stirling.software.proprietary.policy.webhook.WebhookSpool;
/** Push source: deliveries staged to an S3 connection or the local spool, read via the ledger. */
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class WebhookInputSource implements InputSource {
static final String TYPE = "webhook";
private final WebhookSpool spool;
private final FileReadinessChecker readinessChecker;
// Connection-backed staging delegates to the S3 source; else the local spool.
private final S3InputSource s3InputSource;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(InputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
@Override
public void validate(InputSpec spec) {
WebhookConfig config = WebhookConfig.from(spec.options());
if (config.usesConnection()) {
// Resolve with the caller present so save fails if they can't use the connection.
s3InputSource.validate(stagingSpec(config));
}
}
/** The webhook's durable staging as an {@code s3} input spec for {@link S3InputSource}. */
private static InputSpec stagingSpec(WebhookConfig config) {
return new InputSpec(
"s3",
Map.of(
WebhookConfig.CONNECTION_ID_OPTION,
config.connectionId(),
"prefix",
config.stagingPrefix(),
"mode",
config.mode()));
}
/** Mint the routing id + signing secret on create; an existing webhook is left untouched. */
@Override
public Map<String, Object> prepareOptionsForSave(
Map<String, Object> options, boolean isCreate) {
boolean hasId =
options.get(WebhookConfig.WEBHOOK_ID_OPTION) != null
&& !options.get(WebhookConfig.WEBHOOK_ID_OPTION).toString().isBlank();
if (!isCreate && hasId) {
return options;
}
Map<String, Object> prepared = new LinkedHashMap<>(options);
if (!hasId) {
prepared.put(WebhookConfig.WEBHOOK_ID_OPTION, WebhookIds.newWebhookId());
}
Object secret = prepared.get(WebhookConfig.SIGNING_SECRET_OPTION);
if (secret == null || secret.toString().isBlank()) {
prepared.put(WebhookConfig.SIGNING_SECRET_OPTION, WebhookIds.newSigningSecret());
}
return prepared;
}
@Override
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
WebhookConfig config = WebhookConfig.from(spec.options());
if (config.usesConnection()) {
// Durable staging read by the S3 source (no principal; the save-time check is trusted).
return s3InputSource.resolve(stagingSpec(config), ctx);
}
Path dir = spool.dirFor(config.webhookId());
if (!Files.isDirectory(dir)) {
// No deliveries yet: verifiably empty (a missing dir is normal here, not an error).
ctx.reportPresent(List.of());
return List.of();
}
Path canonicalDir = FolderIdentities.canonicalDir(dir);
List<Path> present = listFiles(dir);
if (config.snapshot()) {
List<ResolvedInput> work = new ArrayList<>();
for (Path file : present) {
if (readinessChecker.isReady(file)) {
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
}
}
return work;
}
ctx.reportPresent(
present.stream()
.map(file -> FolderIdentities.identity(canonicalDir, dir, file))
.toList());
List<ResolvedInput> work = new ArrayList<>();
for (Path file : present) {
if (!readinessChecker.isReady(file)) {
continue;
}
String identity = FolderIdentities.identity(canonicalDir, dir, file);
String gate;
boolean claimed;
try {
gate = FolderIdentities.statGate(file);
claimed = ctx.claim(identity, gate, null);
} catch (IOException | UncheckedIOException e) {
log.debug("Could not read {} for its version: {}", file, e.getMessage());
continue; // vanished or unreadable mid-sweep; the next sweep sees the truth
}
if (!claimed) {
continue;
}
work.add(
new ResolvedInput(
PolicyInputs.of(List.of(fileResource(file))),
success -> completeConsumed(ctx, identity, file, gate, success)));
}
return work;
}
/** Settle, then delete when unchanged and every claimant settled DONE (consensus). */
private static void completeConsumed(
ResolveContext ctx, String identity, Path file, String claimGate, boolean success) {
ctx.settle(identity, claimGate, null, success);
if (!success) {
return;
}
try {
if (FolderIdentities.statGate(file).equals(claimGate) && ctx.allSettledDone(identity)) {
Files.deleteIfExists(file);
}
} catch (java.nio.file.NoSuchFileException alreadyGone) {
// Removed by the user or a co-watching policy's own consensus delete: nothing to do.
} catch (IOException e) {
log.warn("Could not remove consumed webhook delivery {}: {}", file, e.getMessage());
}
}
/** Every non-hidden regular file currently spooled for the webhook. */
private static List<Path> listFiles(Path dir) throws IOException {
List<Path> files = new ArrayList<>();
try (Stream<Path> entries = Files.list(dir)) {
entries.filter(Files::isRegularFile)
.filter(file -> !file.getFileName().toString().startsWith("."))
.forEach(files::add);
}
return files;
}
private static Resource fileResource(Path path) {
String name = WebhookSpool.displayName(path.getFileName().toString());
return new FileSystemResource(path.toFile()) {
@Override
public String getFilename() {
return name;
}
};
}
}
@@ -2,6 +2,7 @@ package stirling.software.proprietary.policy.source;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.http.HttpStatus;
@@ -46,6 +47,8 @@ import stirling.software.proprietary.util.SecretMasker;
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class SourceController {
private static final String WEBHOOK_TYPE = "webhook";
private final SourceStore sourceStore;
private final SourceAccessGuard sourceAccessGuard;
private final SourceOverviewService overviewService;
@@ -109,7 +112,8 @@ public class SourceController {
public ResponseEntity<Source> save(@RequestBody Source source) {
requireSourceEditingAllowed();
requireNotEditor(source.id(), source.type());
Source owned = withStoredSecrets(resolveOwnership(source));
boolean isCreate = source.id() == null || source.id().isBlank();
Source owned = withPreparedOptions(withStoredSecrets(resolveOwnership(source)), isCreate);
try {
validateConfig(owned);
} catch (IllegalArgumentException e) {
@@ -119,7 +123,7 @@ public class SourceController {
// An edited folder source can change which directory needs watching, so re-sync trigger
// registrations now instead of waiting for the next reconcile.
policyTriggerManager.notifyPoliciesChanged();
return ResponseEntity.ok(withMaskedSecrets(saved));
return ResponseEntity.ok(revealOnCreate(saved, isCreate));
}
@DeleteMapping("/{sourceId}")
@@ -221,14 +225,36 @@ public class SourceController {
/** Validate the config against the bean that handles the source's type, as the engine will. */
private void validateConfig(Source source) {
InputSpec spec = source.toInputSpec();
inputSources.stream()
.filter(inputSource -> inputSource.supports(spec))
.findFirst()
inputSourceFor(spec)
.orElseThrow(
() -> new IllegalArgumentException("unknown source type: " + source.type()))
.validate(spec);
}
/** Let the source type populate server-owned config before persistence. */
private Source withPreparedOptions(Source source, boolean isCreate) {
InputSpec spec = source.toInputSpec();
InputSource input = inputSourceFor(spec).orElse(null);
if (input == null) {
return source;
}
Map<String, Object> prepared = input.prepareOptionsForSave(source.options(), isCreate);
// Treat a null return as "unchanged" (else the record wipes config to empty).
return prepared == null ? source : withOptions(source, prepared);
}
/** Reveal a webhook's minted secret once on the create response; other reads mask. */
private static Source revealOnCreate(Source saved, boolean isCreate) {
if (isCreate && WEBHOOK_TYPE.equals(saved.type())) {
return saved;
}
return withMaskedSecrets(saved);
}
private Optional<InputSource> inputSourceFor(InputSpec spec) {
return inputSources.stream().filter(input -> input.supports(spec)).findFirst();
}
/**
* Editing sources requires the editor role for the caller's team (a team leader on SaaS), the
* same rule as policies. Single-user deployments (login disabled) trust the local operator.
@@ -102,7 +102,8 @@ public class SourceOverviewService {
List.of(),
docs.total(),
docs.last24h(),
docs.last30d());
docs.last30d(),
null);
}
/**
@@ -143,7 +144,17 @@ public class SourceOverviewService {
configRows(source),
docs.total(),
docs.last24h(),
docs.last30d());
docs.last30d(),
webhookPath(source));
}
/** The server-relative delivery path for a webhook source, else null. Never a secret. */
private static String webhookPath(Source source) {
if (!"webhook".equals(source.type())) {
return null;
}
Object webhookId = source.options().get("webhookId");
return webhookId == null ? null : "/api/v1/webhooks/" + webhookId;
}
/** A disabled (paused) source reads as "disabled"; an unreferenced one reads as "unused". */
@@ -2,11 +2,7 @@ package stirling.software.proprietary.policy.source;
import java.util.List;
/**
* One row in the Sources overview: a persisted input connection shown exactly once, with how many
* policies reference it (and which) and how many documents it has fed into runs ({@code docsTotal}
* lifetime plus the trailing 24-hour and 30-day windows).
*/
/** One Sources-overview row; {@code webhookPath} is a webhook's delivery path, else null. */
public record SourceView(
String id,
String name,
@@ -17,7 +13,8 @@ public record SourceView(
List<DetailRow> config,
long docsTotal,
long docs24h,
long docs30d) {
long docs30d,
String webhookPath) {
/** A policy that references this source. */
public record PolicyRef(String id, String name) {}
@@ -0,0 +1,137 @@
package stirling.software.proprietary.policy.trigger;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.SweepKind;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.webhook.WebhookConfig;
/** Fires webhook policies on delivery plus a periodic reconcile safety net. */
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class WebhookTrigger implements PolicyTrigger {
static final String TYPE = "webhook";
// The webhook source type (equals the trigger type but a distinct concept).
private static final String WEBHOOK_SOURCE_TYPE = "webhook";
private final PolicyStore policyStore;
private final PolicyRunner policyRunner;
private final SourceStore sourceStore;
private final ApplicationProperties applicationProperties;
private volatile ScheduledExecutorService reconciler;
@Override
public String type() {
return TYPE;
}
@Override
public boolean requiresSource() {
return true;
}
@Override
public Set<String> supportedSourceTypes() {
return Set.of(WEBHOOK_SOURCE_TYPE);
}
@Override
public void validate(Policy policy) {
boolean hasWebhookSource =
policy.sourceIds().stream()
.map(sourceStore::get)
.flatMap(java.util.Optional::stream)
.anyMatch(source -> WEBHOOK_SOURCE_TYPE.equals(source.type()));
if (!hasWebhookSource) {
throw new IllegalArgumentException(
"webhook trigger requires at least one webhook input source");
}
}
@Override
public synchronized void start() {
if (reconciler != null) {
return;
}
long reconcileSeconds = applicationProperties.getPolicies().getWatchReconcileSeconds();
reconciler =
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-webhook-reconcile-", 0).factory());
// First reconcile runs immediately so deliveries spooled before startup are picked up.
reconciler.scheduleAtFixedRate(this::safeReconcile, 0, reconcileSeconds, TimeUnit.SECONDS);
log.info("Webhook trigger started (reconcile every {}s)", reconcileSeconds);
}
@Override
public synchronized void stop() {
if (reconciler != null) {
reconciler.shutdownNow();
reconciler = null;
}
}
/** Run every webhook policy referencing the delivered-to source (LIGHT, best-effort). */
public void fireForWebhook(String webhookId) {
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
if (!referencesWebhook(policy, webhookId)) {
continue;
}
try {
log.debug("Webhook policy {} ({}) saw a delivery", policy.id(), policy.name());
policyRunner.run(policy, SweepKind.LIGHT);
} catch (RuntimeException e) {
log.warn("Webhook run failed for policy {}: {}", policy.id(), e.getMessage());
}
}
}
private void safeReconcile() {
try {
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
try {
policyRunner.run(policy);
} catch (RuntimeException e) {
log.warn(
"Webhook reconcile run failed for policy {}: {}",
policy.id(),
e.getMessage());
}
}
} catch (RuntimeException e) {
log.error("Webhook reconcile failed: {}", e.getMessage(), e);
}
}
/** Whether the policy references a webhook source whose routing id is {@code webhookId}. */
private boolean referencesWebhook(Policy policy, String webhookId) {
for (String sourceId : policy.sourceIds()) {
Source source = sourceStore.get(sourceId).orElse(null);
if (source == null || !WEBHOOK_SOURCE_TYPE.equals(source.type())) {
continue;
}
Object configured = source.options().get(WebhookConfig.WEBHOOK_ID_OPTION);
if (configured != null && configured.toString().equals(webhookId)) {
return true;
}
}
return false;
}
}
@@ -0,0 +1,89 @@
package stirling.software.proprietary.policy.webhook;
import java.util.Map;
/** Webhook source config: server-minted ids, staging via an S3 connection or the local spool. */
public record WebhookConfig(
String webhookId, String signingSecret, Long connectionId, boolean snapshot) {
public static final String WEBHOOK_ID_OPTION = "webhookId";
public static final String SIGNING_SECRET_OPTION = "signingSecret";
public static final String CONNECTION_ID_OPTION = "connectionId";
private static final String MODE_OPTION = "mode";
private static final String MODE_CONSUME = "consume";
private static final String MODE_SNAPSHOT = "snapshot";
/** Reserved key namespace webhook deliveries are staged under in a connection's bucket. */
private static final String STAGING_ROOT = "stirling-webhook";
public static WebhookConfig from(Map<String, Object> options) {
String webhookId = trimmed(options.get(WEBHOOK_ID_OPTION));
if (webhookId == null) {
throw new IllegalArgumentException("webhook config requires a 'webhookId' option");
}
if (!WebhookIds.isValidId(webhookId)) {
throw new IllegalArgumentException("webhook config 'webhookId' has an invalid format");
}
String signingSecret = trimmed(options.get(SIGNING_SECRET_OPTION));
if (signingSecret == null) {
throw new IllegalArgumentException("webhook config requires a 'signingSecret' option");
}
String mode = trimmed(options.get(MODE_OPTION));
if (mode != null && !MODE_CONSUME.equals(mode) && !MODE_SNAPSHOT.equals(mode)) {
throw new IllegalArgumentException(
"webhook config 'mode' must be 'consume' or 'snapshot'");
}
return new WebhookConfig(
webhookId, signingSecret, connectionId(options), MODE_SNAPSHOT.equals(mode));
}
/** Whether deliveries are staged to a durable S3 connection rather than the local spool. */
public boolean usesConnection() {
return connectionId != null;
}
/** Reserved per-webhook staging prefix inside the connection's bucket. */
public String stagingPrefix() {
return STAGING_ROOT + "/" + webhookId;
}
public String mode() {
return snapshot ? MODE_SNAPSHOT : MODE_CONSUME;
}
private static Long connectionId(Map<String, Object> options) {
Object reference = options.get(CONNECTION_ID_OPTION);
if (reference == null || (reference instanceof String s && s.isBlank())) {
return null;
}
if (reference instanceof Number number) {
return number.longValue();
}
try {
return Long.valueOf(reference.toString().trim());
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"webhook 'connectionId' is not a valid connection reference: " + reference);
}
}
private static String trimmed(Object value) {
if (value == null) {
return null;
}
String text = value.toString().trim();
return text.isEmpty() ? null : text;
}
/** Never prints the signing secret, so an accidental log line cannot leak it. */
@Override
public String toString() {
return "WebhookConfig[webhookId="
+ webhookId
+ ", connectionId="
+ connectionId
+ ", snapshot="
+ snapshot
+ "]";
}
}
@@ -0,0 +1,38 @@
package stirling.software.proprietary.policy.webhook;
import java.security.SecureRandom;
import java.util.Base64;
import java.util.regex.Pattern;
/** Generates and validates a webhook's routing id and signing secret. */
public final class WebhookIds {
/** URL-safe base64 without padding: the characters a single path segment allows. */
private static final Pattern VALID_ID = Pattern.compile("^[A-Za-z0-9_-]{16,128}$");
private static final SecureRandom RANDOM = new SecureRandom();
private static final Base64.Encoder ENCODER = Base64.getUrlEncoder().withoutPadding();
private WebhookIds() {}
/** A fresh routing token (~24 chars), unguessable so it cannot be enumerated. */
public static String newWebhookId() {
return randomToken(18);
}
/** A fresh HMAC signing key (~43 chars) revealed to the operator once at creation. */
public static String newSigningSecret() {
return randomToken(32);
}
/** Whether {@code id} is well-formed and safe as a path segment / directory name. */
public static boolean isValidId(String id) {
return id != null && VALID_ID.matcher(id).matches();
}
private static String randomToken(int bytes) {
byte[] buffer = new byte[bytes];
RANDOM.nextBytes(buffer);
return ENCODER.encodeToString(buffer);
}
}
@@ -0,0 +1,201 @@
package stirling.software.proprietary.policy.webhook;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.s3.S3Config;
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.trigger.WebhookTrigger;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
/** Public receiver: HMAC-verifies a signed delivery, stages it, and fires the policies. */
@Slf4j
@RestController
@RequestMapping("/api/v1/webhooks")
@Hidden
@RequiredArgsConstructor
@Tag(name = "Webhooks", description = "Inbound webhook source receiver")
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class WebhookReceiverController {
static final String SIGNATURE_HEADER = "X-Stirling-Signature";
static final String FILENAME_HEADER = "X-Stirling-Filename";
private static final String WEBHOOK_TYPE = "webhook";
private final SourceStore sourceStore;
private final WebhookSpool spool;
private final WebhookTrigger webhookTrigger;
private final ApplicationProperties applicationProperties;
private final S3ConnectionResolver connectionResolver;
private final S3ConnectionPool connectionPool;
@PostMapping("/{webhookId}")
@Operation(
summary = "Deliver a document to a webhook source",
description =
"The body is the raw document; sign it with the source's secret and present"
+ " 'sha256=<hex>' in the X-Stirling-Signature header. Returns 202 once"
+ " the document is spooled for the referencing policies.")
public ResponseEntity<WebhookDeliveryResponse> receive(
@PathVariable String webhookId,
@RequestHeader(value = SIGNATURE_HEADER, required = false) String signature,
@RequestHeader(value = FILENAME_HEADER, required = false) String filename,
HttpServletRequest request) {
if (!WebhookIds.isValidId(webhookId)) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No such webhook");
}
Source source = findWebhookSource(webhookId);
if (source == null) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No such webhook");
}
WebhookConfig config = WebhookConfig.from(source.options());
byte[] body = readBoundedBody(request);
if (!WebhookSignatures.verify(config.signingSecret(), body, signature)) {
// Same 401 whether the header was absent or wrong: never confirm a guess.
throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "Invalid signature");
}
if (!source.enabled()) {
throw new ResponseStatusException(
HttpStatus.FORBIDDEN, "Webhook source is paused; deliveries are not accepted");
}
if (body.length == 0) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Empty request body");
}
String storedName =
config.usesConnection()
? stageToConnection(config, filename, body)
: stageToSpool(webhookId, filename, body);
// Fire the referencing policies now; the trigger's reconcile is the safety net.
webhookTrigger.fireForWebhook(webhookId);
log.info(
"Accepted webhook delivery '{}' ({} bytes) for {}",
storedName,
body.length,
webhookId);
return ResponseEntity.accepted()
.contentType(MediaType.APPLICATION_JSON)
.body(new WebhookDeliveryResponse(true, storedName, body.length));
}
/** The enabled-or-not webhook source whose routing id matches, or null if there is none. */
private Source findWebhookSource(String webhookId) {
for (Source source : sourceStore.all()) {
if (!WEBHOOK_TYPE.equals(source.type())) {
continue;
}
Object configured = source.options().get(WebhookConfig.WEBHOOK_ID_OPTION);
if (configured != null && configured.toString().equals(webhookId)) {
return source;
}
}
return null;
}
/** Stage a delivery to the node-local spool; returns its display (original) name. */
private String stageToSpool(String webhookId, String filename, byte[] body) {
try {
return WebhookSpool.displayName(
spool.store(webhookId, filename, body).getFileName().toString());
} catch (IOException e) {
log.error("Could not spool webhook delivery for {}: {}", webhookId, e.getMessage());
throw new ResponseStatusException(
HttpStatus.INTERNAL_SERVER_ERROR, "Could not store delivery");
}
}
/** Stage a delivery to the S3 connection (save-time access check trusted). */
private String stageToConnection(WebhookConfig config, String filename, byte[] body) {
S3Config s3 =
connectionResolver.resolve(
Map.of(
WebhookConfig.CONNECTION_ID_OPTION,
config.connectionId(),
"prefix",
config.stagingPrefix()));
S3Client client = connectionPool.clientFor(s3);
String key = keyPrefix(s3.prefix()) + WebhookSpool.objectKeySuffix(filename);
try {
client.putObject(
PutObjectRequest.builder().bucket(s3.bucket()).key(key).build(),
RequestBody.fromBytes(body));
} catch (SdkException e) {
log.error(
"Could not stage webhook delivery for {} to s3://{}/{}: {}",
config.webhookId(),
s3.bucket(),
key,
e.getMessage());
throw new ResponseStatusException(
HttpStatus.INTERNAL_SERVER_ERROR, "Could not store delivery");
}
return WebhookSpool.objectDisplayName(filename);
}
/** The configured prefix as a key-path prefix ("inbox" and "inbox/" mean the same). */
private static String keyPrefix(String prefix) {
if (prefix == null || prefix.isEmpty() || prefix.endsWith("/")) {
return prefix == null ? "" : prefix;
}
return prefix + "/";
}
/** Read the body into memory, capped at {@code policies.webhookMaxBytes}. */
private byte[] readBoundedBody(HttpServletRequest request) {
long maxBytes = applicationProperties.getPolicies().getWebhookMaxBytes();
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
byte[] chunk = new byte[8192];
long total = 0;
try (InputStream in = request.getInputStream()) {
int read;
while ((read = in.read(chunk)) != -1) {
total += read;
if (total > maxBytes) {
throw new ResponseStatusException(
HttpStatus.PAYLOAD_TOO_LARGE,
"Delivery exceeds the " + maxBytes + "-byte limit");
}
buffer.write(chunk, 0, read);
}
} catch (IOException e) {
throw new ResponseStatusException(
HttpStatus.BAD_REQUEST, "Could not read request body");
}
return buffer.toByteArray();
}
/** The 202 body: the stored (display) name and byte count of an accepted delivery. */
public record WebhookDeliveryResponse(boolean accepted, String filename, int bytes) {}
}
@@ -0,0 +1,53 @@
package stirling.software.proprietary.policy.webhook;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
/** HMAC-SHA256 signing of a webhook delivery's raw body (header sha256=<hex>). */
public final class WebhookSignatures {
private static final String ALGORITHM = "HmacSHA256";
private static final String PREFIX = "sha256=";
private WebhookSignatures() {}
/** The header a sender presents for {@code body}: {@code sha256=<lowercase hex>}. */
public static String sign(String signingSecret, byte[] body) {
return PREFIX + HexFormat.of().formatHex(hmac(signingSecret, body));
}
/** Whether the presented signature is valid for {@code body}; false on bad input. */
public static boolean verify(String signingSecret, byte[] body, String presented) {
if (signingSecret == null || presented == null || body == null) {
return false;
}
String hex = presented.trim();
if (hex.regionMatches(true, 0, PREFIX, 0, PREFIX.length())) {
hex = hex.substring(PREFIX.length());
}
byte[] presentedBytes;
try {
presentedBytes = HexFormat.of().parseHex(hex);
} catch (IllegalArgumentException notHex) {
return false;
}
return MessageDigest.isEqual(hmac(signingSecret, body), presentedBytes);
}
private static byte[] hmac(String signingSecret, byte[] body) {
try {
Mac mac = Mac.getInstance(ALGORITHM);
mac.init(new SecretKeySpec(signingSecret.getBytes(StandardCharsets.UTF_8), ALGORITHM));
return mac.doFinal(body);
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
// HmacSHA256 is a required JCE algorithm and the key is always non-empty here.
throw new IllegalStateException("HMAC-SHA256 unavailable", e);
}
}
}
@@ -0,0 +1,106 @@
package stirling.software.proprietary.policy.webhook;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.UUID;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.stereotype.Component;
import stirling.software.common.configuration.InstallationPathConfig;
/** Server-owned per-webhook staging directory under the install path. */
@Component
@ConditionalOnBooleanProperty(name = "policies.enabled")
public class WebhookSpool {
private static final String SPOOL_DIR = "policy-webhook-spool";
private static final String TEMP_SUFFIX = ".part";
private static final String DEFAULT_NAME = "document.pdf";
private static final int UNIQUE_LEN = 32; // UUID hex without dashes
private final Path spoolRoot;
public WebhookSpool() {
this(Path.of(InstallationPathConfig.getPath(), SPOOL_DIR));
}
// Lets a caller (and tests) root the spool at a chosen directory; Spring uses the no-arg one.
public WebhookSpool(Path spoolRoot) {
this.spoolRoot = spoolRoot.toAbsolutePath().normalize();
}
/** The staging directory for one webhook. Never escapes the spool root; may not yet exist. */
public Path dirFor(String webhookId) {
if (!WebhookIds.isValidId(webhookId)) {
throw new IllegalArgumentException("invalid webhookId");
}
Path dir = spoolRoot.resolve(webhookId).normalize();
if (!dir.getParent().equals(spoolRoot)) {
// A validated id is a single safe segment; this only trips on a bug, never user input.
throw new IllegalArgumentException("invalid webhookId");
}
return dir;
}
/** Write a delivery into the spool atomically (staged .part, then moved into place). */
public Path store(String webhookId, String filename, byte[] content) throws IOException {
Path dir = dirFor(webhookId);
Files.createDirectories(dir);
String finalName = spoolName(filename);
Path target = dir.resolve(finalName);
Path temp = dir.resolve("." + finalName + TEMP_SUFFIX);
Files.write(temp, content);
try {
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException atomicUnsupported) {
Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING);
}
return target;
}
/** The spool file name for a delivery: a unique prefix plus the sanitised original name. */
static String spoolName(String filename) {
return UUID.randomUUID().toString().replace("-", "") + "-" + sanitize(filename);
}
/** The original name recovered from a spool file name, for the resolved input's filename. */
public static String displayName(String spoolFileName) {
int dash = spoolFileName.indexOf('-');
// The unique prefix is fixed-length hex with no dashes, so the first dash is the separator.
if (dash == UNIQUE_LEN && dash + 1 < spoolFileName.length()) {
return spoolFileName.substring(dash + 1);
}
return spoolFileName;
}
/** S3 object-key suffix {@code <unique>/<name>}; the subfolder keeps the basename clean. */
public static String objectKeySuffix(String filename) {
return UUID.randomUUID().toString().replace("-", "") + "/" + sanitize(filename);
}
/** The staged delivery's display name (basename of {@link #objectKeySuffix}). */
public static String objectDisplayName(String filename) {
return sanitize(filename);
}
/** Reduce a client-supplied filename to a safe bare basename; fall back to a default. */
private static String sanitize(String filename) {
if (filename == null) {
return DEFAULT_NAME;
}
String base = filename.replace('\\', '/');
int slash = base.lastIndexOf('/');
if (slash >= 0) {
base = base.substring(slash + 1);
}
base = base.replaceAll("[^A-Za-z0-9._-]", "_").trim();
// Strip leading dots so the result is never hidden (which resolve would skip) or empty.
while (base.startsWith(".")) {
base = base.substring(1);
}
return base.isEmpty() ? DEFAULT_NAME : base;
}
}
@@ -20,9 +20,8 @@ public final class SecretMasker {
private static final Pattern SENSITIVE =
RegexPatternUtils.getInstance()
.getPattern(
// secret[_-]?access[_-]?key precedes plain secret so camelCase keys
// like secretAccessKey (no word boundary after "secret") still match.
"(?i)\\b(password|token|secret[_-]?access[_-]?key|secret|api[_-]?key|authorization|auth|jwt|cred|cert)\\b");
// specific keys precede plain "secret" so camelCase still matches.
"(?i)\\b(password|token|secret[_-]?access[_-]?key|signing[_-]?secret|secret|api[_-]?key|authorization|auth|jwt|cred|cert)\\b");
private SecretMasker() {}
@@ -0,0 +1,221 @@
package stirling.software.proprietary.policy.input;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.webhook.WebhookConfig;
import stirling.software.proprietary.policy.webhook.WebhookSpool;
/** Tests for {@link WebhookInputSource}: ledger-backed read/consume, and id/secret minting. */
@ExtendWith(MockitoExtension.class)
class WebhookInputSourceTest {
private static final String POLICY = "p1";
private static final String WEBHOOK_ID = "testwebhookid1234";
@Mock private FileReadinessChecker readinessChecker;
@Mock private S3InputSource s3InputSource;
@TempDir Path tempDir;
private WebhookSpool spool;
private WebhookInputSource source;
private InProcessProcessedLedger ledger;
private RecordingContext ctx;
@BeforeEach
void setUp() {
spool = new WebhookSpool(tempDir.resolve("spool"));
source = new WebhookInputSource(spool, readinessChecker, s3InputSource);
ledger = new InProcessProcessedLedger();
ctx = new RecordingContext();
lenient().when(readinessChecker.isReady(any())).thenReturn(true);
}
private static InputSpec spec(String mode) {
return new InputSpec(
"webhook",
Map.of("webhookId", WEBHOOK_ID, "signingSecret", "secret", "mode", mode));
}
@Test
void consumeRemovesTheDeliveryOnceProcessed() throws IOException {
Path delivered = spool.store(WEBHOOK_ID, "doc.pdf", "data".getBytes());
List<ResolvedInput> work = source.resolve(spec("consume"), ctx);
assertEquals(1, work.size());
assertEquals("doc.pdf", work.get(0).inputs().primary().get(0).getFilename());
// In flight: still spooled, but a second sweep does not pick it up again.
assertTrue(Files.exists(delivered));
assertTrue(source.resolve(spec("consume"), ctx).isEmpty());
work.get(0).onComplete().accept(true);
assertTrue(Files.notExists(delivered));
assertTrue(source.resolve(spec("consume"), ctx).isEmpty());
}
@Test
void aFailedRunLeavesTheDeliveryInPlace() throws IOException {
Path delivered = spool.store(WEBHOOK_ID, "doc.pdf", "data".getBytes());
List<ResolvedInput> work = source.resolve(spec("consume"), ctx);
work.get(0).onComplete().accept(false);
assertTrue(Files.exists(delivered));
}
@Test
void snapshotReReadsEveryRunAndNeverDeletes() throws IOException {
Path delivered = spool.store(WEBHOOK_ID, "doc.pdf", "data".getBytes());
assertEquals(1, source.resolve(spec("snapshot"), ctx).size());
List<ResolvedInput> second = source.resolve(spec("snapshot"), ctx);
assertEquals(1, second.size());
second.get(0).onComplete().accept(true);
assertTrue(Files.exists(delivered));
}
@Test
void nothingDeliveredIsAnEmptySourceNotAnError() throws IOException {
List<ResolvedInput> work = source.resolve(spec("consume"), ctx);
assertTrue(work.isEmpty());
assertTrue(ctx.present.isEmpty());
}
@Test
void validateRejectsMissingIdOrSecret() {
assertThrows(
IllegalArgumentException.class,
() -> source.validate(new InputSpec("webhook", Map.of("signingSecret", "s"))));
assertThrows(
IllegalArgumentException.class,
() -> source.validate(new InputSpec("webhook", Map.of("webhookId", WEBHOOK_ID))));
}
@Test
void prepareMintsIdAndSecretOnCreate() {
Map<String, Object> prepared =
source.prepareOptionsForSave(Map.of("mode", "consume"), true);
String id = prepared.get(WebhookConfig.WEBHOOK_ID_OPTION).toString();
String secret = prepared.get(WebhookConfig.SIGNING_SECRET_OPTION).toString();
assertFalse(id.isBlank());
assertFalse(secret.isBlank());
assertEquals("consume", prepared.get("mode"));
Map<String, Object> other = source.prepareOptionsForSave(Map.of(), true);
assertNotEquals(id, other.get(WebhookConfig.WEBHOOK_ID_OPTION).toString());
}
@Test
void prepareLeavesAnExistingWebhookUntouchedOnEdit() {
Map<String, Object> existing =
Map.of("webhookId", WEBHOOK_ID, "signingSecret", "keepme", "mode", "snapshot");
Map<String, Object> prepared = source.prepareOptionsForSave(existing, false);
assertEquals(WEBHOOK_ID, prepared.get("webhookId"));
assertEquals("keepme", prepared.get("signingSecret"));
}
@Test
void aConnectionBackedWebhookDelegatesToTheS3SourceUnderItsReservedPrefix() throws IOException {
when(s3InputSource.resolve(any(), any())).thenReturn(List.of());
InputSpec spec =
new InputSpec(
"webhook",
Map.of(
"webhookId",
WEBHOOK_ID,
"signingSecret",
"secret",
"mode",
"consume",
"connectionId",
7));
source.resolve(spec, ctx);
ArgumentCaptor<InputSpec> delegated = ArgumentCaptor.forClass(InputSpec.class);
verify(s3InputSource).resolve(delegated.capture(), eq(ctx));
InputSpec s3 = delegated.getValue();
assertEquals("s3", s3.type());
assertEquals(7L, ((Number) s3.options().get("connectionId")).longValue());
assertEquals("stirling-webhook/" + WEBHOOK_ID, s3.options().get("prefix"));
assertEquals("consume", s3.options().get("mode"));
}
@Test
void aConnectionBackedWebhookValidatesTheConnectionThroughTheS3Source() {
InputSpec spec =
new InputSpec(
"webhook",
Map.of(
"webhookId",
WEBHOOK_ID,
"signingSecret",
"secret",
"connectionId",
7));
source.validate(spec);
// Save-time validation defers to the S3 source, which ownership-checks the connection.
verify(s3InputSource).validate(any());
}
/** Policy-scoped context backed by the in-process ledger, recording presence reports. */
private class RecordingContext implements ResolveContext {
private final List<String> present = new ArrayList<>();
@Override
public boolean claim(String identity, String gate, Supplier<String> contentHash) {
return ledger.claim(POLICY, identity, gate, contentHash);
}
@Override
public void settle(
String identity, String finalGate, String finalContentHash, boolean success) {
ledger.settle(POLICY, identity, finalGate, finalContentHash, success);
}
@Override
public boolean allSettledDone(String identity) {
return ledger.allSettledDone(identity);
}
@Override
public void reportPresent(Collection<String> identities) {
present.addAll(identities);
}
}
}
@@ -1,33 +1,42 @@
package stirling.software.proprietary.policy.source;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.http.ResponseEntity;
import org.springframework.web.server.ResponseStatusException;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.service.UserServiceInterface;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.input.S3InputSource;
import stirling.software.proprietary.policy.input.WebhookInputSource;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
import stirling.software.proprietary.policy.webhook.WebhookSpool;
import stirling.software.proprietary.util.SecretMasker;
/**
@@ -41,6 +50,9 @@ class SourceControllerTest {
private final PolicyStore policyStore = new InProcessPolicyStore();
private PolicyTriggerManager triggerManager;
private SourceController controller;
private SourceController webhookController;
@TempDir Path tempDir;
@BeforeEach
void setUp() {
@@ -58,9 +70,11 @@ class SourceControllerTest {
policyGuard,
new InProcessSourceDocCounter());
triggerManager = mock(PolicyTriggerManager.class);
// A permissive input source so config validation passes and save can be exercised.
// A permissive input source; stub prepareOptionsForSave as a pass-through.
InputSource folderInput = mock(InputSource.class);
when(folderInput.supports(any())).thenReturn(true);
when(folderInput.prepareOptionsForSave(any(), anyBoolean()))
.thenAnswer(invocation -> invocation.getArgument(0));
controller =
new SourceController(
sourceStore,
@@ -72,6 +86,48 @@ class SourceControllerTest {
triggerManager,
properties,
List.of(folderInput));
WebhookInputSource webhookInput =
new WebhookInputSource(
new WebhookSpool(tempDir),
mock(FileReadinessChecker.class),
mock(S3InputSource.class));
webhookController =
new SourceController(
sourceStore,
sourceGuard,
overviewService,
policyStore,
policyGuard,
authority,
triggerManager,
properties,
List.of(webhookInput));
}
@Test
void creatingAWebhookRevealsItsSecretOnceThenMasks() {
Source created =
webhookController
.save(
new Source(
null,
"Partner uploads",
"webhook",
Map.of("mode", "consume"),
true,
null,
null))
.getBody();
String secret = String.valueOf(created.options().get("signingSecret"));
String webhookId = String.valueOf(created.options().get("webhookId"));
assertNotEquals(SecretMasker.REDACTED, secret);
assertFalse(secret.isBlank());
assertFalse(webhookId.isBlank());
Source read = webhookController.get(created.id()).getBody();
assertEquals(SecretMasker.REDACTED, read.options().get("signingSecret"));
assertEquals(webhookId, read.options().get("webhookId"));
}
@Test
@@ -0,0 +1,117 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.SweepKind;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.source.InProcessSourceStore;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.store.PolicyStore;
/** Tests for {@link WebhookTrigger}: fires only policies referencing the delivered-to webhook. */
@ExtendWith(MockitoExtension.class)
class WebhookTriggerTest {
private static final String TYPE = "webhook";
@Mock private PolicyStore policyStore;
@Mock private PolicyRunner policyRunner;
private final SourceStore sourceStore = new InProcessSourceStore();
private WebhookTrigger trigger;
@BeforeEach
void setUp() {
trigger =
new WebhookTrigger(
policyStore, policyRunner, sourceStore, new ApplicationProperties());
}
@Test
void firesOnlyPoliciesReferencingTheDeliveredWebhook() {
Policy matching = webhookPolicy("a", "whkA");
Policy other = webhookPolicy("b", "whkB");
when(policyStore.findByTriggerType(TYPE)).thenReturn(List.of(matching, other));
trigger.fireForWebhook("whkA");
verify(policyRunner).run(matching, SweepKind.LIGHT);
verify(policyRunner, never()).run(other, SweepKind.LIGHT);
}
@Test
void ignoresADeliveryForAnUnknownWebhookId() {
Policy policy = webhookPolicy("a", "whkA");
when(policyStore.findByTriggerType(TYPE)).thenReturn(List.of(policy));
trigger.fireForWebhook("whkZ");
verify(policyRunner, never()).run(any(), any(SweepKind.class));
}
@Test
void validateRequiresAWebhookSource() {
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(policy("p", webhookTriggerConfig(), List.of())));
trigger.validate(webhookPolicy("p", "whkA"));
}
private static TriggerConfig webhookTriggerConfig() {
return new TriggerConfig(TYPE, Map.of());
}
/** Persist a webhook source with the given routing id and return a policy referencing it. */
private Policy webhookPolicy(String id, String webhookId) {
String sourceId =
sourceStore
.save(
new Source(
null,
"hook",
"webhook",
Map.of(
"webhookId",
webhookId,
"signingSecret",
"s",
"mode",
"consume"),
true,
"owner",
null))
.id();
return policy(id, webhookTriggerConfig(), List.of(sourceId));
}
private static Policy policy(String id, TriggerConfig trigger, List<String> sourceIds) {
return new Policy(
id,
"hook",
"owner",
true,
trigger,
sourceIds,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
@@ -0,0 +1,186 @@
package stirling.software.proprietary.policy.webhook;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.web.server.ResponseStatusException;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
import stirling.software.proprietary.policy.source.InProcessSourceStore;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.source.SourceStore;
import stirling.software.proprietary.policy.trigger.WebhookTrigger;
import stirling.software.proprietary.policy.webhook.WebhookReceiverController.WebhookDeliveryResponse;
/** Tests for the public webhook receiver: valid delivery spools + fires; bad requests rejected. */
class WebhookReceiverControllerTest {
private static final String WEBHOOK_ID = "receivertestid12";
private static final String SECRET = "topsecret";
private static final byte[] BODY = "a pdf".getBytes(StandardCharsets.UTF_8);
@TempDir Path tempDir;
private SourceStore sourceStore;
private WebhookSpool spool;
private WebhookTrigger trigger;
private ApplicationProperties properties;
private WebhookReceiverController controller;
@BeforeEach
void setUp() {
sourceStore = new InProcessSourceStore();
sourceStore.save(webhookSource(true));
spool = new WebhookSpool(tempDir.resolve("spool"));
trigger = mock(WebhookTrigger.class);
properties = new ApplicationProperties();
// The local-disk tests never touch a connection; mocks stand in for the S3 collaborators.
controller =
new WebhookReceiverController(
sourceStore,
spool,
trigger,
properties,
mock(S3ConnectionResolver.class),
mock(S3ConnectionPool.class));
}
private static Source webhookSource(boolean enabled) {
return new Source(
"s1",
"Partner uploads",
"webhook",
Map.of("webhookId", WEBHOOK_ID, "signingSecret", SECRET, "mode", "consume"),
enabled,
"owner",
null);
}
private static MockHttpServletRequest request(byte[] body) {
MockHttpServletRequest req =
new MockHttpServletRequest("POST", "/api/v1/webhooks/" + WEBHOOK_ID);
req.setContent(body);
return req;
}
@Test
void aValidDeliveryIsSpooledAndFiresTheTrigger() throws IOException {
String signature = WebhookSignatures.sign(SECRET, BODY);
ResponseEntity<WebhookDeliveryResponse> response =
controller.receive(WEBHOOK_ID, signature, "invoice.pdf", request(BODY));
assertEquals(202, response.getStatusCode().value());
assertTrue(response.getBody().accepted());
assertEquals("invoice.pdf", response.getBody().filename());
assertEquals(1, spooledFiles().size());
verify(trigger).fireForWebhook(WEBHOOK_ID);
}
@Test
void aWrongSignatureIsRejectedAndStoresNothing() {
ResponseStatusException ex =
assertThrows(
ResponseStatusException.class,
() ->
controller.receive(
WEBHOOK_ID, "sha256=deadbeef", "x.pdf", request(BODY)));
assertEquals(401, ex.getStatusCode().value());
assertTrue(spooledFiles().isEmpty());
verify(trigger, never()).fireForWebhook(WEBHOOK_ID);
}
@Test
void anUnknownWebhookIsNotFound() {
ResponseStatusException ex =
assertThrows(
ResponseStatusException.class,
() ->
controller.receive(
"unknownwebhookid",
WebhookSignatures.sign(SECRET, BODY),
"x.pdf",
request(BODY)));
assertEquals(404, ex.getStatusCode().value());
}
@Test
void aPausedSourceRejectsDeliveries() {
sourceStore.save(webhookSource(false));
String signature = WebhookSignatures.sign(SECRET, BODY);
ResponseStatusException ex =
assertThrows(
ResponseStatusException.class,
() -> controller.receive(WEBHOOK_ID, signature, "x.pdf", request(BODY)));
assertEquals(403, ex.getStatusCode().value());
assertTrue(spooledFiles().isEmpty());
}
@Test
void anEmptyBodyIsRejected() {
byte[] empty = new byte[0];
String signature = WebhookSignatures.sign(SECRET, empty);
ResponseStatusException ex =
assertThrows(
ResponseStatusException.class,
() -> controller.receive(WEBHOOK_ID, signature, null, request(empty)));
assertEquals(400, ex.getStatusCode().value());
}
@Test
void anOversizeDeliveryIsRejectedBeforeStoring() {
properties.getPolicies().setWebhookMaxBytes(2);
ResponseStatusException ex =
assertThrows(
ResponseStatusException.class,
() ->
controller.receive(
WEBHOOK_ID,
WebhookSignatures.sign(SECRET, BODY),
"x.pdf",
request(BODY)));
assertEquals(413, ex.getStatusCode().value());
assertTrue(spooledFiles().isEmpty());
}
private List<Path> spooledFiles() {
Path dir = spool.dirFor(WEBHOOK_ID);
if (!Files.isDirectory(dir)) {
return List.of();
}
try (Stream<Path> entries = Files.list(dir)) {
return entries.filter(Files::isRegularFile)
.filter(p -> !p.getFileName().toString().startsWith("."))
.toList();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@@ -0,0 +1,263 @@
package stirling.software.proprietary.policy.webhook;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.mock.web.MockHttpServletRequest;
import org.testcontainers.containers.MinIOContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.access.service.OwnershipService;
import stirling.software.proprietary.integration.model.IntegrationConfig;
import stirling.software.proprietary.integration.model.IntegrationType;
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
import stirling.software.proprietary.policy.input.ResolveContext;
import stirling.software.proprietary.policy.input.ResolvedInput;
import stirling.software.proprietary.policy.input.S3InputSource;
import stirling.software.proprietary.policy.input.WebhookInputSource;
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
import stirling.software.proprietary.policy.source.InProcessSourceStore;
import stirling.software.proprietary.policy.source.Source;
import stirling.software.proprietary.policy.trigger.WebhookTrigger;
import stirling.software.proprietary.security.service.UserService;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.S3Object;
import tools.jackson.databind.ObjectMapper;
/** End-to-end connection-backed webhook against MinIO: stage, read via S3 delegation, consume. */
@Testcontainers(disabledWithoutDocker = true)
class WebhookS3ConnectionMinioTest {
private static final String POLICY = "p1";
private static final String ACCESS_KEY = "minioadmin";
private static final String SECRET_KEY = "minioadmin";
private static final String WEBHOOK_ID = "miniowebhookid12";
private static final String SECRET = "topsecret";
private static final long CONNECTION_ID = 7L;
@Container
static MinIOContainer minio =
new MinIOContainer("minio/minio:latest")
.withUserName(ACCESS_KEY)
.withPassword(SECRET_KEY);
private static S3Client adminClient;
private static int bucketCounter;
@TempDir Path tempDir;
private String bucket;
private String stagingPrefix;
private WebhookReceiverController receiver;
private WebhookInputSource inputSource;
private InProcessProcessedLedger ledger;
private RecordingContext ctx;
@BeforeEach
void setUp() {
if (adminClient == null) {
adminClient =
S3Client.builder()
.endpointOverride(URI.create(minio.getS3URL()))
.httpClient(UrlConnectionHttpClient.create())
.region(Region.US_EAST_1)
.credentialsProvider(
StaticCredentialsProvider.create(
AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
.serviceConfiguration(
S3Configuration.builder().pathStyleAccessEnabled(true).build())
.build();
}
bucket = "webhook-inbox-" + ++bucketCounter;
adminClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
stagingPrefix = "stirling-webhook/" + WEBHOOK_ID + "/";
// The MinIO endpoint resolves to loopback, so the operator opt-in must be on.
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowPrivateS3Endpoints(true);
// A resolver over a MinIO S3 connection; no principal here, so ownership is skipped.
S3ConnectionResolver resolver = resolverFor(minioConnection());
S3ConnectionPool pool = new S3ConnectionPool(properties);
S3InputSource s3 = new S3InputSource(pool, resolver);
inputSource =
new WebhookInputSource(
new WebhookSpool(tempDir.resolve("spool")),
mock(FileReadinessChecker.class),
s3);
InProcessSourceStore sourceStore = new InProcessSourceStore();
sourceStore.save(
new Source(
"s1",
"Partner uploads",
"webhook",
Map.of(
"webhookId", WEBHOOK_ID,
"signingSecret", SECRET,
"mode", "consume",
"connectionId", CONNECTION_ID),
true,
"owner",
null));
receiver =
new WebhookReceiverController(
sourceStore,
new WebhookSpool(tempDir.resolve("spool")),
mock(WebhookTrigger.class),
properties,
resolver,
pool);
ledger = new InProcessProcessedLedger();
ctx = new RecordingContext();
}
@Test
void aDeliveryIsStagedToTheConnectionThenReadAndConsumed() throws IOException {
byte[] body = "a pdf".getBytes(StandardCharsets.UTF_8);
String signature = WebhookSignatures.sign(SECRET, body);
var response = receiver.receive(WEBHOOK_ID, signature, "invoice.pdf", request(body));
assertThat(response.getStatusCode().value()).isEqualTo(202);
List<S3Object> staged = listUnder(stagingPrefix);
assertThat(staged).hasSize(1);
// Staged under a unique subfolder so the object basename stays the clean original name.
assertThat(staged.get(0).key()).endsWith("/invoice.pdf");
List<ResolvedInput> work = inputSource.resolve(webhookSpec(), ctx);
assertThat(work).hasSize(1);
assertThat(work.get(0).inputs().primary().get(0).getFilename()).isEqualTo("invoice.pdf");
assertThat(read(work.get(0))).isEqualTo("a pdf");
// In flight: a second sweep claims nothing.
assertThat(inputSource.resolve(webhookSpec(), ctx)).isEmpty();
work.get(0).onComplete().accept(true);
assertThat(listUnder(stagingPrefix)).isEmpty();
assertThat(inputSource.resolve(webhookSpec(), ctx)).isEmpty();
}
@Test
void aRejectedDeliveryStagesNothing() {
byte[] body = "a pdf".getBytes(StandardCharsets.UTF_8);
try {
receiver.receive(WEBHOOK_ID, "sha256=deadbeef", "x.pdf", request(body));
} catch (RuntimeException expected) {
// 401 invalid signature
}
assertThat(listUnder(stagingPrefix)).isEmpty();
}
private IntegrationConfig minioConnection() {
IntegrationConfig connection = new IntegrationConfig();
connection.setId(CONNECTION_ID);
connection.setIntegrationType(IntegrationType.S3);
connection.setName("minio");
connection.setEnabled(true);
connection.setConfig(
new ObjectMapper()
.writeValueAsString(
Map.of(
"bucket", bucket,
"region", "us-east-1",
"endpoint", minio.getS3URL(),
"accessKeyId", ACCESS_KEY,
"secretAccessKey", SECRET_KEY)));
return connection;
}
private static S3ConnectionResolver resolverFor(IntegrationConfig connection) {
IntegrationConfigRepository connections = mock(IntegrationConfigRepository.class);
when(connections.findById(connection.getId())).thenReturn(Optional.of(connection));
return new S3ConnectionResolver(
connections, mock(OwnershipService.class), mock(UserService.class));
}
private static InputSpec webhookSpec() {
return new InputSpec(
"webhook",
Map.of(
"webhookId", WEBHOOK_ID,
"signingSecret", SECRET,
"mode", "consume",
"connectionId", CONNECTION_ID));
}
private static MockHttpServletRequest request(byte[] body) {
MockHttpServletRequest req =
new MockHttpServletRequest("POST", "/api/v1/webhooks/" + WEBHOOK_ID);
req.setContent(body);
return req;
}
private List<S3Object> listUnder(String prefix) {
return adminClient
.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).prefix(prefix).build())
.contents();
}
private static String read(ResolvedInput unit) throws IOException {
try (InputStream stream = unit.inputs().primary().get(0).getInputStream()) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
}
}
private class RecordingContext implements ResolveContext {
private final List<String> present = new ArrayList<>();
@Override
public boolean claim(String identity, String gate, Supplier<String> contentHash) {
return ledger.claim(POLICY, identity, gate, contentHash);
}
@Override
public void settle(
String identity, String finalGate, String finalContentHash, boolean success) {
ledger.settle(POLICY, identity, finalGate, finalContentHash, success);
}
@Override
public boolean allSettledDone(String identity) {
return ledger.allSettledDone(identity);
}
@Override
public void reportPresent(Collection<String> identities) {
present.addAll(identities);
}
}
}
@@ -0,0 +1,49 @@
package stirling.software.proprietary.policy.webhook;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.Test;
/** The HMAC-SHA256 webhook signing scheme: correct signatures verify, tampered ones do not. */
class WebhookSignaturesTest {
private static final String SECRET = "whsec_test_secret";
private static final byte[] BODY = "the document bytes".getBytes(StandardCharsets.UTF_8);
@Test
void aFreshlySignedBodyVerifies() {
String header = WebhookSignatures.sign(SECRET, BODY);
assertTrue(header.startsWith("sha256="));
assertTrue(WebhookSignatures.verify(SECRET, BODY, header));
}
@Test
void abarehexSignatureVerifiesToo() {
String header = WebhookSignatures.sign(SECRET, BODY);
String bareHex = header.substring("sha256=".length());
assertTrue(WebhookSignatures.verify(SECRET, BODY, bareHex));
}
@Test
void aWrongSecretDoesNotVerify() {
String header = WebhookSignatures.sign(SECRET, BODY);
assertFalse(WebhookSignatures.verify("other-secret", BODY, header));
}
@Test
void atamperedBodyDoesNotVerify() {
String header = WebhookSignatures.sign(SECRET, BODY);
byte[] tampered = "the document byteS".getBytes(StandardCharsets.UTF_8);
assertFalse(WebhookSignatures.verify(SECRET, tampered, header));
}
@Test
void aMissingOrMalformedHeaderIsFalseNotAnError() {
assertFalse(WebhookSignatures.verify(SECRET, BODY, null));
assertFalse(WebhookSignatures.verify(SECRET, BODY, "sha256=not-hex"));
assertFalse(WebhookSignatures.verify(SECRET, BODY, ""));
}
}
@@ -103,6 +103,18 @@ class SecretMaskerTest {
assertEquals("AKIAEXAMPLE", result.get("accessKeyId"));
}
@Test
@DisplayName("should mask camelCase signingSecret despite no word boundary")
void shouldMaskCamelCaseSigningSecret() {
Map<String, Object> input = Map.of("signingSecret", "shh", "webhookId", "whk_abc");
Map<String, Object> result = SecretMasker.mask(input);
assertEquals(SecretMasker.REDACTED, result.get("signingSecret"));
// The routing id is a public URL token, not a secret.
assertEquals("whk_abc", result.get("webhookId"));
}
@Test
@DisplayName("should mask nested map sensitive keys")
void shouldMaskNestedMapSensitiveKeys() {
@@ -8139,6 +8139,36 @@ placeholder = "us-east-1"
[portal.sources.types.s3.fields.secretAccessKey]
label = "Secret access key"
[portal.sources.types.webhook]
description = "Receive documents by signed HTTP POST from any external system."
label = "Webhook"
[portal.sources.types.webhook.detail]
deliveryUrl = "Delivery URL"
secretNote = "The signing secret is shown once, when the webhook is created. Recreate the source to roll it."
[portal.sources.types.webhook.fields.connection]
helperText = "Choose an S3 connection to stage deliveries durably (required in hosted). Leave blank on a self-hosted server to stage them on local disk."
label = "Storage connection"
[portal.sources.types.webhook.fields.mode]
helperText = "Consume removes each delivered document once every policy has processed it."
label = "Read mode"
[portal.sources.types.webhook.fields.mode.options]
consume = "Consume: process each delivery once"
snapshot = "Snapshot: re-read the spool every run"
[portal.sources.types.webhook.reveal]
copy = "Copy"
done = "Done"
secret = "Signing secret"
secretHelp = "Sign each delivery's raw body with this key (HMAC-SHA256) and send it as the X-Stirling-Signature header."
secretWarning = "Copy the signing secret now. For your security it is shown only once and cannot be retrieved later."
title = "Webhook created"
url = "Delivery URL"
usage = "POST each document as the raw request body to the delivery URL with a binary content type (application/pdf or application/octet-stream). Referencing policies run automatically on arrival."
[portal.sources.types.unknown]
label = "Source"
@@ -7,6 +7,15 @@ describe("creatableSourceTypes (SaaS)", () => {
expect(creatableSourceTypes().map((t) => t.type)).not.toContain("folder");
});
it("offers webhook with a required S3 connection (durable staging in hosted)", () => {
const webhook = creatableSourceTypes().find((t) => t.type === "webhook");
expect(webhook).toBeDefined();
const connection = webhook?.fields.find(
(f) => f.control === "s3Connection",
);
expect(connection?.required).toBe(true);
});
it("still offers the cloud source types", () => {
expect(creatableSourceTypes().map((t) => t.type)).toContain("s3");
});
@@ -1,13 +1,25 @@
import {
CREATABLE_SOURCE_TYPES,
WEBHOOK_SOURCE_TYPE,
type CreatableSourceType,
} from "@portal/components/sources/sourceTypes";
/**
* Hosted deployments never read the server's filesystem (the backend's
* FolderAccessGuard denies it outright), so folder connections are not offered
* in the connect wizard.
*/
/** Hosted drops folder (no server FS) and requires webhook's S3 connection (node-local isn't durable). */
export function creatableSourceTypes(): CreatableSourceType[] {
return CREATABLE_SOURCE_TYPES.filter((type) => type.type !== "folder");
return CREATABLE_SOURCE_TYPES.filter((type) => type.type !== "folder").map(
(type) =>
type.type === WEBHOOK_SOURCE_TYPE ? withRequiredConnection(type) : type,
);
}
/** The type with its S3-connection field marked required (durable staging is mandatory). */
function withRequiredConnection(
type: CreatableSourceType,
): CreatableSourceType {
return {
...type,
fields: type.fields.map((field) =>
field.control === "s3Connection" ? { ...field, required: true } : field,
),
};
}
@@ -32,6 +32,8 @@ export interface SourceView {
docsTotal: number;
docs24h: number;
docs30d: number;
/** Webhook sources only: the server-relative delivery path senders POST to. Null otherwise. */
webhookPath?: string | null;
}
export interface SourceKpi {
@@ -22,6 +22,9 @@ export interface SourceTypeMeta {
*/
export const EDITOR_SOURCE_TYPE = "editor";
/** The webhook source type. Its delivery URL + signing secret are minted server-side on create. */
export const WEBHOOK_SOURCE_TYPE = "webhook";
const SOURCE_TYPE_META: Record<string, SourceTypeMeta> = {
folder: {
labelKey: "portal.sources.types.folder.label",
@@ -38,6 +41,11 @@ const SOURCE_TYPE_META: Record<string, SourceTypeMeta> = {
icon: "☁",
accent: "brand",
},
webhook: {
labelKey: "portal.sources.types.webhook.label",
icon: "↯",
accent: "warning",
},
};
const UNKNOWN_TYPE_META: SourceTypeMeta = {
@@ -180,6 +188,40 @@ export const CREATABLE_SOURCE_TYPES: CreatableSourceType[] = [
},
],
},
{
// URL + secret minted server-side (revealed once); an S3 connection makes staging durable, else local disk.
type: WEBHOOK_SOURCE_TYPE,
labelKey: "portal.sources.types.webhook.label",
descriptionKey: "portal.sources.types.webhook.description",
fields: [
{
key: "connectionId",
labelKey: "portal.sources.types.webhook.fields.connection.label",
control: "s3Connection",
helperTextKey:
"portal.sources.types.webhook.fields.connection.helperText",
},
{
key: "mode",
labelKey: "portal.sources.types.webhook.fields.mode.label",
control: "select",
defaultValue: "consume",
helperTextKey: "portal.sources.types.webhook.fields.mode.helperText",
options: [
{
value: "consume",
labelKey:
"portal.sources.types.webhook.fields.mode.options.consume",
},
{
value: "snapshot",
labelKey:
"portal.sources.types.webhook.fields.mode.options.snapshot",
},
],
},
],
},
];
/** Default option values for a type's create form. */
@@ -55,6 +55,18 @@ function seedSources(): StoredSource[] {
enabled: false,
owner: "data-eng@acme.com",
},
{
id: "src-webhook",
name: "Partner uploads",
type: "webhook",
options: {
webhookId: "whk_demo_5f3a9c21b7",
signingSecret: "whsec_demo_2b8e1d47a9f60c35",
mode: "consume",
},
enabled: true,
owner: "you@acme.com",
},
];
}
@@ -76,6 +88,7 @@ const docCounts: Record<
"src-contracts": { total: 12840, last24h: 96, last30d: 2310 },
"src-archive": { total: 1180, last24h: 0, last30d: 0 },
"src-legacy": { total: 48600, last24h: 0, last30d: 0 },
"src-webhook": { total: 3120, last24h: 24, last30d: 640 },
};
function docsFor(id: string): {
@@ -96,14 +109,23 @@ function nextId(): string {
return `src_${Date.now().toString(36)}_${idCounter}`;
}
/** A demo token for a newly created webhook's server-generated id/secret (mock only). */
function randomToken(): string {
return (
Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2)
);
}
function refsFor(id: string): SourcePolicyRef[] {
return references[id] ?? [];
}
/** Mirror the backend's secret redaction so config rows never surface a secret in the overview. */
function configRows(options: Record<string, unknown>) {
const secret = /secret|password|token/i;
return Object.entries(options).map(([key, value]) => ({
label: key.charAt(0).toUpperCase() + key.slice(1),
value: String(value),
value: secret.test(key) ? "********" : String(value),
}));
}
@@ -120,6 +142,7 @@ function toSourceView(
refs: SourcePolicyRef[],
): SourceView {
const docs = docsFor(source.id);
const webhookId = source.options.webhookId;
return {
id: source.id,
name: source.name,
@@ -131,6 +154,10 @@ function toSourceView(
docsTotal: docs.total,
docs24h: docs.last24h,
docs30d: docs.last30d,
webhookPath:
source.type === "webhook" && typeof webhookId === "string"
? `/api/v1/webhooks/${webhookId}`
: null,
};
}
@@ -207,8 +234,18 @@ export const sourcesHandlers = [
? store.find((s) => s.id === incoming.id)
: undefined;
const id = existing?.id ?? nextId();
// A new webhook's id + secret are minted server-side and revealed once on this create response.
let options = incoming.options ?? {};
if (incoming.type === "webhook" && !existing && !options.webhookId) {
options = {
...options,
webhookId: `whk_${randomToken()}`,
signingSecret: `whsec_${randomToken()}`,
};
}
const saved: StoredSource = {
...incoming,
options,
id,
owner: existing?.owner ?? "you@acme.com",
};
@@ -85,3 +85,27 @@
justify-content: flex-end;
gap: 0.5rem;
}
/* Webhook delivery URL / signing secret: read-only value + copy button. */
.portal-source-builder__copy-row {
display: flex;
gap: 0.5rem;
align-items: center;
}
.portal-source-builder__copy-row > :first-child {
flex: 1 1 auto;
min-width: 0;
}
.portal-source-builder__reveal {
display: flex;
flex-direction: column;
gap: 1rem;
}
.portal-source-builder__muted {
margin: 0;
font-size: 0.8125rem;
color: var(--color-text-4);
}
@@ -102,6 +102,39 @@ describe("SourceBuilder", () => {
).toBeDisabled();
});
it("reveals the delivery URL and signing secret once after creating a webhook", async () => {
createSource.mockResolvedValue({
id: "wh-1",
options: { webhookId: "whk_abc123", signingSecret: "whsec_topsecret" },
});
renderBuilder("/processor/sources/new");
fireEvent.change(screen.getByLabelText(/portal\.sources\.wizard\.name/), {
target: { value: "Partner uploads" },
});
// Webhook's connection is optional (self-hosted local-disk), so a name is enough to create.
fireEvent.click(screen.getByText("portal.sources.types.webhook.label"));
fireEvent.click(screen.getByText("portal.sources.builder.create"));
await waitFor(() => expect(createSource).toHaveBeenCalledTimes(1));
expect(createSource).toHaveBeenCalledWith(
expect.objectContaining({ type: "webhook", name: "Partner uploads" }),
);
expect(
await screen.findByDisplayValue("whsec_topsecret"),
).toBeInTheDocument();
expect(
screen.getByDisplayValue(/\/api\/v1\/webhooks\/whk_abc123$/),
).toBeInTheDocument();
expect(screen.queryByText("sources list")).not.toBeInTheDocument();
fireEvent.click(
screen.getByText("portal.sources.types.webhook.reveal.done"),
);
expect(await screen.findByText("sources list")).toBeInTheDocument();
});
it("blocks create until required fields are filled", async () => {
renderBuilder("/processor/sources/new");
// Name given but directory (required) still blank -> Create disabled.
@@ -27,11 +27,17 @@ import {
CREATABLE_SOURCE_TYPES,
defaultOptions,
sourceTypeMeta,
WEBHOOK_SOURCE_TYPE,
type CreatableSourceType,
} from "@portal/components/sources/sourceTypes";
import { S3ConnectionPicker } from "@portal/components/sources/S3ConnectionPicker";
import "@portal/views/SourceBuilder.css";
/** Absolute delivery URL a sender POSTs to for a webhook's routing id. */
function webhookUrl(webhookId: string): string {
return `${window.location.origin}/api/v1/webhooks/${webhookId}`;
}
const OFFERED_TYPES = creatableSourceTypes();
/** A source's stored type resolved to its create-form metadata (edit falls back to any type). */
@@ -84,6 +90,11 @@ export function SourceBuilder() {
const [error, setError] = useState<string | null>(null);
const [pendingDelete, setPendingDelete] = useState(false);
const [deleting, setDeleting] = useState(false);
// Set after a webhook is created: its one-time delivery id + signing secret, shown before leaving.
const [reveal, setReveal] = useState<{
webhookId: string;
secret: string;
} | null>(null);
// Seed once: immediately for a new source, or after the record loads for edit.
useEffect(() => {
@@ -112,18 +123,40 @@ export function SourceBuilder() {
);
const canSave = name.trim() !== "" && requiredComplete && !submitting;
// An existing webhook's delivery URL, read-only in edit (the secret is revealed only at creation).
const editingWebhookId =
isEdit && sourceState.data?.type === WEBHOOK_SOURCE_TYPE
? String(sourceState.data.options?.webhookId ?? "")
: "";
const revealUrl = reveal ? webhookUrl(reveal.webhookId) : "";
const revealSecret = reveal ? reveal.secret : "";
function dismissReveal() {
setReveal(null);
navigate(listPath);
}
async function save() {
if (!canSave) return;
setSubmitting(true);
setError(null);
try {
await createSource({
const saved = await createSource({
id: isEdit ? id : undefined,
name: name.trim(),
type: type.type,
options,
enabled,
});
// A new webhook returns its minted id + secret once; reveal them (with the URL) before leaving.
if (!isEdit && type.type === WEBHOOK_SOURCE_TYPE) {
const webhookId = String(saved.options?.webhookId ?? "");
const secret = String(saved.options?.signingSecret ?? "");
if (webhookId && secret) {
setReveal({ webhookId, secret });
return;
}
}
navigate(listPath);
} catch (e) {
setError(errorMessage(e));
@@ -131,6 +164,10 @@ export function SourceBuilder() {
}
}
function copy(text: string) {
void navigator.clipboard?.writeText(text);
}
async function confirmDelete() {
if (!id || deleting) return;
setDeleting(true);
@@ -290,6 +327,28 @@ export function SourceBuilder() {
</FormField>
))}
{editingWebhookId && (
<FormField
label={t("portal.sources.types.webhook.detail.deliveryUrl")}
helperText={t("portal.sources.types.webhook.detail.secretNote")}
>
<div className="portal-source-builder__copy-row">
<Input
value={webhookUrl(editingWebhookId)}
readOnly
onFocus={(e) => e.currentTarget.select()}
/>
<Button
variant="tertiary"
size="sm"
onClick={() => copy(webhookUrl(editingWebhookId))}
>
{t("portal.sources.types.webhook.reveal.copy")}
</Button>
</div>
</FormField>
)}
{error && <Banner tone="danger" description={error} />}
</div>
@@ -321,6 +380,69 @@ export function SourceBuilder() {
>
<p>{t("portal.sources.delete.body", { name })}</p>
</Modal>
<Modal
open={reveal !== null}
onClose={dismissReveal}
width="md"
title={t("portal.sources.types.webhook.reveal.title")}
footer={
<div className="portal-source-builder__delete-actions">
<Button size="sm" onClick={dismissReveal}>
{t("portal.sources.types.webhook.reveal.done")}
</Button>
</div>
}
>
{reveal && (
<div className="portal-source-builder__reveal">
<Banner
tone="warning"
description={t(
"portal.sources.types.webhook.reveal.secretWarning",
)}
/>
<FormField label={t("portal.sources.types.webhook.reveal.url")}>
<div className="portal-source-builder__copy-row">
<Input
value={revealUrl}
readOnly
onFocus={(e) => e.currentTarget.select()}
/>
<Button
variant="tertiary"
size="sm"
onClick={() => copy(revealUrl)}
>
{t("portal.sources.types.webhook.reveal.copy")}
</Button>
</div>
</FormField>
<FormField
label={t("portal.sources.types.webhook.reveal.secret")}
helperText={t("portal.sources.types.webhook.reveal.secretHelp")}
>
<div className="portal-source-builder__copy-row">
<Input
value={revealSecret}
readOnly
onFocus={(e) => e.currentTarget.select()}
/>
<Button
variant="tertiary"
size="sm"
onClick={() => copy(revealSecret)}
>
{t("portal.sources.types.webhook.reveal.copy")}
</Button>
</div>
</FormField>
<p className="portal-source-builder__muted">
{t("portal.sources.types.webhook.reveal.usage")}
</p>
</div>
)}
</Modal>
</div>
);
}