Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f6bc5e956 | ||
|
|
cb40d7778f | ||
|
|
8cb03dba9e | ||
|
|
247b783c75 | ||
|
|
a9f7add87d |
@@ -254,6 +254,9 @@ public class ApplicationProperties {
|
||||
* in-network object store.
|
||||
*/
|
||||
private boolean allowPrivateS3Endpoints = false;
|
||||
|
||||
/** Max inbound webhook body (bytes); over it is rejected 413. Default 100 MB. */
|
||||
private long webhookMaxBytes = 104857600L;
|
||||
}
|
||||
|
||||
@Data
|
||||
|
||||
@@ -198,6 +198,8 @@ public class RequestUriUtils {
|
||||
|| trimmedUri.startsWith("/readiness")
|
||||
|| trimmedUri.startsWith(
|
||||
"/api/v1/mobile-scanner/") // Mobile scanner endpoints (no auth)
|
||||
// Webhook receiver: authenticated per-request by HMAC signature, not a session.
|
||||
|| trimmedUri.startsWith("/api/v1/webhooks/")
|
||||
|| trimmedUri.startsWith("/v1/api-docs")
|
||||
// Workflow participant endpoints - access controlled by share tokens, not login
|
||||
|| trimmedUri.startsWith("/api/v1/workflow/participant/")
|
||||
|
||||
@@ -176,6 +176,13 @@ class RequestUriUtilsTest {
|
||||
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/api/v1/convert", ""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIsPublicAuthEndpoint_webhookReceiver() {
|
||||
// The webhook source receiver authenticates each delivery by HMAC signature, not a session.
|
||||
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/api/v1/webhooks/whk_abc123", ""));
|
||||
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/app/api/v1/webhooks/whk_abc123", "/app"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIsPublicAuthEndpoint_withContextPath() {
|
||||
assertTrue(RequestUriUtils.isPublicAuthEndpoint("/app/login", "/app"));
|
||||
|
||||
+7
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.input;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
|
||||
@@ -22,6 +23,12 @@ public interface InputSource {
|
||||
/** Throws {@link IllegalArgumentException} on bad config. Called on save to fail fast. */
|
||||
default void validate(InputSpec spec) {}
|
||||
|
||||
/** Normalise a source's options before persistence (default no-op); must not mutate the arg. */
|
||||
default Map<String, Object> prepareOptionsForSave(
|
||||
Map<String, Object> options, boolean isCreate) {
|
||||
return options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the spec into zero or more units of work, each carrying one run's files and a
|
||||
* completion hook. Empty list means nothing to run right now. Discovery is read-only - files
|
||||
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.policy.ledger.FolderIdentities;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookConfig;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookIds;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookSpool;
|
||||
|
||||
/** Push source: deliveries staged to an S3 connection or the local spool, read via the ledger. */
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class WebhookInputSource implements InputSource {
|
||||
|
||||
static final String TYPE = "webhook";
|
||||
|
||||
private final WebhookSpool spool;
|
||||
private final FileReadinessChecker readinessChecker;
|
||||
// Connection-backed staging delegates to the S3 source; else the local spool.
|
||||
private final S3InputSource s3InputSource;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(InputSpec spec) {
|
||||
return spec != null && TYPE.equals(spec.type());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(InputSpec spec) {
|
||||
WebhookConfig config = WebhookConfig.from(spec.options());
|
||||
if (config.usesConnection()) {
|
||||
// Resolve with the caller present so save fails if they can't use the connection.
|
||||
s3InputSource.validate(stagingSpec(config));
|
||||
}
|
||||
}
|
||||
|
||||
/** The webhook's durable staging as an {@code s3} input spec for {@link S3InputSource}. */
|
||||
private static InputSpec stagingSpec(WebhookConfig config) {
|
||||
return new InputSpec(
|
||||
"s3",
|
||||
Map.of(
|
||||
WebhookConfig.CONNECTION_ID_OPTION,
|
||||
config.connectionId(),
|
||||
"prefix",
|
||||
config.stagingPrefix(),
|
||||
"mode",
|
||||
config.mode()));
|
||||
}
|
||||
|
||||
/** Mint the routing id + signing secret on create; an existing webhook is left untouched. */
|
||||
@Override
|
||||
public Map<String, Object> prepareOptionsForSave(
|
||||
Map<String, Object> options, boolean isCreate) {
|
||||
boolean hasId =
|
||||
options.get(WebhookConfig.WEBHOOK_ID_OPTION) != null
|
||||
&& !options.get(WebhookConfig.WEBHOOK_ID_OPTION).toString().isBlank();
|
||||
if (!isCreate && hasId) {
|
||||
return options;
|
||||
}
|
||||
Map<String, Object> prepared = new LinkedHashMap<>(options);
|
||||
if (!hasId) {
|
||||
prepared.put(WebhookConfig.WEBHOOK_ID_OPTION, WebhookIds.newWebhookId());
|
||||
}
|
||||
Object secret = prepared.get(WebhookConfig.SIGNING_SECRET_OPTION);
|
||||
if (secret == null || secret.toString().isBlank()) {
|
||||
prepared.put(WebhookConfig.SIGNING_SECRET_OPTION, WebhookIds.newSigningSecret());
|
||||
}
|
||||
return prepared;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
|
||||
WebhookConfig config = WebhookConfig.from(spec.options());
|
||||
if (config.usesConnection()) {
|
||||
// Durable staging read by the S3 source (no principal; the save-time check is trusted).
|
||||
return s3InputSource.resolve(stagingSpec(config), ctx);
|
||||
}
|
||||
Path dir = spool.dirFor(config.webhookId());
|
||||
if (!Files.isDirectory(dir)) {
|
||||
// No deliveries yet: verifiably empty (a missing dir is normal here, not an error).
|
||||
ctx.reportPresent(List.of());
|
||||
return List.of();
|
||||
}
|
||||
Path canonicalDir = FolderIdentities.canonicalDir(dir);
|
||||
List<Path> present = listFiles(dir);
|
||||
|
||||
if (config.snapshot()) {
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (Path file : present) {
|
||||
if (readinessChecker.isReady(file)) {
|
||||
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
|
||||
}
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
ctx.reportPresent(
|
||||
present.stream()
|
||||
.map(file -> FolderIdentities.identity(canonicalDir, dir, file))
|
||||
.toList());
|
||||
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (Path file : present) {
|
||||
if (!readinessChecker.isReady(file)) {
|
||||
continue;
|
||||
}
|
||||
String identity = FolderIdentities.identity(canonicalDir, dir, file);
|
||||
String gate;
|
||||
boolean claimed;
|
||||
try {
|
||||
gate = FolderIdentities.statGate(file);
|
||||
claimed = ctx.claim(identity, gate, null);
|
||||
} catch (IOException | UncheckedIOException e) {
|
||||
log.debug("Could not read {} for its version: {}", file, e.getMessage());
|
||||
continue; // vanished or unreadable mid-sweep; the next sweep sees the truth
|
||||
}
|
||||
if (!claimed) {
|
||||
continue;
|
||||
}
|
||||
work.add(
|
||||
new ResolvedInput(
|
||||
PolicyInputs.of(List.of(fileResource(file))),
|
||||
success -> completeConsumed(ctx, identity, file, gate, success)));
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
/** Settle, then delete when unchanged and every claimant settled DONE (consensus). */
|
||||
private static void completeConsumed(
|
||||
ResolveContext ctx, String identity, Path file, String claimGate, boolean success) {
|
||||
ctx.settle(identity, claimGate, null, success);
|
||||
if (!success) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (FolderIdentities.statGate(file).equals(claimGate) && ctx.allSettledDone(identity)) {
|
||||
Files.deleteIfExists(file);
|
||||
}
|
||||
} catch (java.nio.file.NoSuchFileException alreadyGone) {
|
||||
// Removed by the user or a co-watching policy's own consensus delete: nothing to do.
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not remove consumed webhook delivery {}: {}", file, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** Every non-hidden regular file currently spooled for the webhook. */
|
||||
private static List<Path> listFiles(Path dir) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
try (Stream<Path> entries = Files.list(dir)) {
|
||||
entries.filter(Files::isRegularFile)
|
||||
.filter(file -> !file.getFileName().toString().startsWith("."))
|
||||
.forEach(files::add);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private static Resource fileResource(Path path) {
|
||||
String name = WebhookSpool.displayName(path.getFileName().toString());
|
||||
return new FileSystemResource(path.toFile()) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return name;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
+31
-5
@@ -2,6 +2,7 @@ package stirling.software.proprietary.policy.source;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@@ -46,6 +47,8 @@ import stirling.software.proprietary.util.SecretMasker;
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class SourceController {
|
||||
|
||||
private static final String WEBHOOK_TYPE = "webhook";
|
||||
|
||||
private final SourceStore sourceStore;
|
||||
private final SourceAccessGuard sourceAccessGuard;
|
||||
private final SourceOverviewService overviewService;
|
||||
@@ -109,7 +112,8 @@ public class SourceController {
|
||||
public ResponseEntity<Source> save(@RequestBody Source source) {
|
||||
requireSourceEditingAllowed();
|
||||
requireNotEditor(source.id(), source.type());
|
||||
Source owned = withStoredSecrets(resolveOwnership(source));
|
||||
boolean isCreate = source.id() == null || source.id().isBlank();
|
||||
Source owned = withPreparedOptions(withStoredSecrets(resolveOwnership(source)), isCreate);
|
||||
try {
|
||||
validateConfig(owned);
|
||||
} catch (IllegalArgumentException e) {
|
||||
@@ -119,7 +123,7 @@ public class SourceController {
|
||||
// An edited folder source can change which directory needs watching, so re-sync trigger
|
||||
// registrations now instead of waiting for the next reconcile.
|
||||
policyTriggerManager.notifyPoliciesChanged();
|
||||
return ResponseEntity.ok(withMaskedSecrets(saved));
|
||||
return ResponseEntity.ok(revealOnCreate(saved, isCreate));
|
||||
}
|
||||
|
||||
@DeleteMapping("/{sourceId}")
|
||||
@@ -221,14 +225,36 @@ public class SourceController {
|
||||
/** Validate the config against the bean that handles the source's type, as the engine will. */
|
||||
private void validateConfig(Source source) {
|
||||
InputSpec spec = source.toInputSpec();
|
||||
inputSources.stream()
|
||||
.filter(inputSource -> inputSource.supports(spec))
|
||||
.findFirst()
|
||||
inputSourceFor(spec)
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException("unknown source type: " + source.type()))
|
||||
.validate(spec);
|
||||
}
|
||||
|
||||
/** Let the source type populate server-owned config before persistence. */
|
||||
private Source withPreparedOptions(Source source, boolean isCreate) {
|
||||
InputSpec spec = source.toInputSpec();
|
||||
InputSource input = inputSourceFor(spec).orElse(null);
|
||||
if (input == null) {
|
||||
return source;
|
||||
}
|
||||
Map<String, Object> prepared = input.prepareOptionsForSave(source.options(), isCreate);
|
||||
// Treat a null return as "unchanged" (else the record wipes config to empty).
|
||||
return prepared == null ? source : withOptions(source, prepared);
|
||||
}
|
||||
|
||||
/** Reveal a webhook's minted secret once on the create response; other reads mask. */
|
||||
private static Source revealOnCreate(Source saved, boolean isCreate) {
|
||||
if (isCreate && WEBHOOK_TYPE.equals(saved.type())) {
|
||||
return saved;
|
||||
}
|
||||
return withMaskedSecrets(saved);
|
||||
}
|
||||
|
||||
private Optional<InputSource> inputSourceFor(InputSpec spec) {
|
||||
return inputSources.stream().filter(input -> input.supports(spec)).findFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Editing sources requires the editor role for the caller's team (a team leader on SaaS), the
|
||||
* same rule as policies. Single-user deployments (login disabled) trust the local operator.
|
||||
|
||||
+13
-2
@@ -102,7 +102,8 @@ public class SourceOverviewService {
|
||||
List.of(),
|
||||
docs.total(),
|
||||
docs.last24h(),
|
||||
docs.last30d());
|
||||
docs.last30d(),
|
||||
null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -143,7 +144,17 @@ public class SourceOverviewService {
|
||||
configRows(source),
|
||||
docs.total(),
|
||||
docs.last24h(),
|
||||
docs.last30d());
|
||||
docs.last30d(),
|
||||
webhookPath(source));
|
||||
}
|
||||
|
||||
/** The server-relative delivery path for a webhook source, else null. Never a secret. */
|
||||
private static String webhookPath(Source source) {
|
||||
if (!"webhook".equals(source.type())) {
|
||||
return null;
|
||||
}
|
||||
Object webhookId = source.options().get("webhookId");
|
||||
return webhookId == null ? null : "/api/v1/webhooks/" + webhookId;
|
||||
}
|
||||
|
||||
/** A disabled (paused) source reads as "disabled"; an unreferenced one reads as "unused". */
|
||||
|
||||
+3
-6
@@ -2,11 +2,7 @@ package stirling.software.proprietary.policy.source;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* One row in the Sources overview: a persisted input connection shown exactly once, with how many
|
||||
* policies reference it (and which) and how many documents it has fed into runs ({@code docsTotal}
|
||||
* lifetime plus the trailing 24-hour and 30-day windows).
|
||||
*/
|
||||
/** One Sources-overview row; {@code webhookPath} is a webhook's delivery path, else null. */
|
||||
public record SourceView(
|
||||
String id,
|
||||
String name,
|
||||
@@ -17,7 +13,8 @@ public record SourceView(
|
||||
List<DetailRow> config,
|
||||
long docsTotal,
|
||||
long docs24h,
|
||||
long docs30d) {
|
||||
long docs30d,
|
||||
String webhookPath) {
|
||||
|
||||
/** A policy that references this source. */
|
||||
public record PolicyRef(String id, String name) {}
|
||||
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
package stirling.software.proprietary.policy.trigger;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookConfig;
|
||||
|
||||
/** Fires webhook policies on delivery plus a periodic reconcile safety net. */
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class WebhookTrigger implements PolicyTrigger {
|
||||
|
||||
static final String TYPE = "webhook";
|
||||
// The webhook source type (equals the trigger type but a distinct concept).
|
||||
private static final String WEBHOOK_SOURCE_TYPE = "webhook";
|
||||
|
||||
private final PolicyStore policyStore;
|
||||
private final PolicyRunner policyRunner;
|
||||
private final SourceStore sourceStore;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
private volatile ScheduledExecutorService reconciler;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requiresSource() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> supportedSourceTypes() {
|
||||
return Set.of(WEBHOOK_SOURCE_TYPE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Policy policy) {
|
||||
boolean hasWebhookSource =
|
||||
policy.sourceIds().stream()
|
||||
.map(sourceStore::get)
|
||||
.flatMap(java.util.Optional::stream)
|
||||
.anyMatch(source -> WEBHOOK_SOURCE_TYPE.equals(source.type()));
|
||||
if (!hasWebhookSource) {
|
||||
throw new IllegalArgumentException(
|
||||
"webhook trigger requires at least one webhook input source");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (reconciler != null) {
|
||||
return;
|
||||
}
|
||||
long reconcileSeconds = applicationProperties.getPolicies().getWatchReconcileSeconds();
|
||||
reconciler =
|
||||
Executors.newSingleThreadScheduledExecutor(
|
||||
Thread.ofVirtual().name("policy-webhook-reconcile-", 0).factory());
|
||||
// First reconcile runs immediately so deliveries spooled before startup are picked up.
|
||||
reconciler.scheduleAtFixedRate(this::safeReconcile, 0, reconcileSeconds, TimeUnit.SECONDS);
|
||||
log.info("Webhook trigger started (reconcile every {}s)", reconcileSeconds);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (reconciler != null) {
|
||||
reconciler.shutdownNow();
|
||||
reconciler = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Run every webhook policy referencing the delivered-to source (LIGHT, best-effort). */
|
||||
public void fireForWebhook(String webhookId) {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
if (!referencesWebhook(policy, webhookId)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
log.debug("Webhook policy {} ({}) saw a delivery", policy.id(), policy.name());
|
||||
policyRunner.run(policy, SweepKind.LIGHT);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("Webhook run failed for policy {}: {}", policy.id(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void safeReconcile() {
|
||||
try {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
try {
|
||||
policyRunner.run(policy);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"Webhook reconcile run failed for policy {}: {}",
|
||||
policy.id(),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
log.error("Webhook reconcile failed: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the policy references a webhook source whose routing id is {@code webhookId}. */
|
||||
private boolean referencesWebhook(Policy policy, String webhookId) {
|
||||
for (String sourceId : policy.sourceIds()) {
|
||||
Source source = sourceStore.get(sourceId).orElse(null);
|
||||
if (source == null || !WEBHOOK_SOURCE_TYPE.equals(source.type())) {
|
||||
continue;
|
||||
}
|
||||
Object configured = source.options().get(WebhookConfig.WEBHOOK_ID_OPTION);
|
||||
if (configured != null && configured.toString().equals(webhookId)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/** Webhook source config: server-minted ids, staging via an S3 connection or the local spool. */
|
||||
public record WebhookConfig(
|
||||
String webhookId, String signingSecret, Long connectionId, boolean snapshot) {
|
||||
|
||||
public static final String WEBHOOK_ID_OPTION = "webhookId";
|
||||
public static final String SIGNING_SECRET_OPTION = "signingSecret";
|
||||
public static final String CONNECTION_ID_OPTION = "connectionId";
|
||||
private static final String MODE_OPTION = "mode";
|
||||
private static final String MODE_CONSUME = "consume";
|
||||
private static final String MODE_SNAPSHOT = "snapshot";
|
||||
|
||||
/** Reserved key namespace webhook deliveries are staged under in a connection's bucket. */
|
||||
private static final String STAGING_ROOT = "stirling-webhook";
|
||||
|
||||
public static WebhookConfig from(Map<String, Object> options) {
|
||||
String webhookId = trimmed(options.get(WEBHOOK_ID_OPTION));
|
||||
if (webhookId == null) {
|
||||
throw new IllegalArgumentException("webhook config requires a 'webhookId' option");
|
||||
}
|
||||
if (!WebhookIds.isValidId(webhookId)) {
|
||||
throw new IllegalArgumentException("webhook config 'webhookId' has an invalid format");
|
||||
}
|
||||
String signingSecret = trimmed(options.get(SIGNING_SECRET_OPTION));
|
||||
if (signingSecret == null) {
|
||||
throw new IllegalArgumentException("webhook config requires a 'signingSecret' option");
|
||||
}
|
||||
String mode = trimmed(options.get(MODE_OPTION));
|
||||
if (mode != null && !MODE_CONSUME.equals(mode) && !MODE_SNAPSHOT.equals(mode)) {
|
||||
throw new IllegalArgumentException(
|
||||
"webhook config 'mode' must be 'consume' or 'snapshot'");
|
||||
}
|
||||
return new WebhookConfig(
|
||||
webhookId, signingSecret, connectionId(options), MODE_SNAPSHOT.equals(mode));
|
||||
}
|
||||
|
||||
/** Whether deliveries are staged to a durable S3 connection rather than the local spool. */
|
||||
public boolean usesConnection() {
|
||||
return connectionId != null;
|
||||
}
|
||||
|
||||
/** Reserved per-webhook staging prefix inside the connection's bucket. */
|
||||
public String stagingPrefix() {
|
||||
return STAGING_ROOT + "/" + webhookId;
|
||||
}
|
||||
|
||||
public String mode() {
|
||||
return snapshot ? MODE_SNAPSHOT : MODE_CONSUME;
|
||||
}
|
||||
|
||||
private static Long connectionId(Map<String, Object> options) {
|
||||
Object reference = options.get(CONNECTION_ID_OPTION);
|
||||
if (reference == null || (reference instanceof String s && s.isBlank())) {
|
||||
return null;
|
||||
}
|
||||
if (reference instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
try {
|
||||
return Long.valueOf(reference.toString().trim());
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException(
|
||||
"webhook 'connectionId' is not a valid connection reference: " + reference);
|
||||
}
|
||||
}
|
||||
|
||||
private static String trimmed(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String text = value.toString().trim();
|
||||
return text.isEmpty() ? null : text;
|
||||
}
|
||||
|
||||
/** Never prints the signing secret, so an accidental log line cannot leak it. */
|
||||
@Override
|
||||
public String toString() {
|
||||
return "WebhookConfig[webhookId="
|
||||
+ webhookId
|
||||
+ ", connectionId="
|
||||
+ connectionId
|
||||
+ ", snapshot="
|
||||
+ snapshot
|
||||
+ "]";
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Base64;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Generates and validates a webhook's routing id and signing secret. */
|
||||
public final class WebhookIds {
|
||||
|
||||
/** URL-safe base64 without padding: the characters a single path segment allows. */
|
||||
private static final Pattern VALID_ID = Pattern.compile("^[A-Za-z0-9_-]{16,128}$");
|
||||
|
||||
private static final SecureRandom RANDOM = new SecureRandom();
|
||||
private static final Base64.Encoder ENCODER = Base64.getUrlEncoder().withoutPadding();
|
||||
|
||||
private WebhookIds() {}
|
||||
|
||||
/** A fresh routing token (~24 chars), unguessable so it cannot be enumerated. */
|
||||
public static String newWebhookId() {
|
||||
return randomToken(18);
|
||||
}
|
||||
|
||||
/** A fresh HMAC signing key (~43 chars) revealed to the operator once at creation. */
|
||||
public static String newSigningSecret() {
|
||||
return randomToken(32);
|
||||
}
|
||||
|
||||
/** Whether {@code id} is well-formed and safe as a path segment / directory name. */
|
||||
public static boolean isValidId(String id) {
|
||||
return id != null && VALID_ID.matcher(id).matches();
|
||||
}
|
||||
|
||||
private static String randomToken(int bytes) {
|
||||
byte[] buffer = new byte[bytes];
|
||||
RANDOM.nextBytes(buffer);
|
||||
return ENCODER.encodeToString(buffer);
|
||||
}
|
||||
}
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.s3.S3Config;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.trigger.WebhookTrigger;
|
||||
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
|
||||
/** Public receiver: HMAC-verifies a signed delivery, stages it, and fires the policies. */
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/webhooks")
|
||||
@Hidden
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Webhooks", description = "Inbound webhook source receiver")
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class WebhookReceiverController {
|
||||
|
||||
static final String SIGNATURE_HEADER = "X-Stirling-Signature";
|
||||
static final String FILENAME_HEADER = "X-Stirling-Filename";
|
||||
private static final String WEBHOOK_TYPE = "webhook";
|
||||
|
||||
private final SourceStore sourceStore;
|
||||
private final WebhookSpool spool;
|
||||
private final WebhookTrigger webhookTrigger;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final S3ConnectionResolver connectionResolver;
|
||||
private final S3ConnectionPool connectionPool;
|
||||
|
||||
@PostMapping("/{webhookId}")
|
||||
@Operation(
|
||||
summary = "Deliver a document to a webhook source",
|
||||
description =
|
||||
"The body is the raw document; sign it with the source's secret and present"
|
||||
+ " 'sha256=<hex>' in the X-Stirling-Signature header. Returns 202 once"
|
||||
+ " the document is spooled for the referencing policies.")
|
||||
public ResponseEntity<WebhookDeliveryResponse> receive(
|
||||
@PathVariable String webhookId,
|
||||
@RequestHeader(value = SIGNATURE_HEADER, required = false) String signature,
|
||||
@RequestHeader(value = FILENAME_HEADER, required = false) String filename,
|
||||
HttpServletRequest request) {
|
||||
if (!WebhookIds.isValidId(webhookId)) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No such webhook");
|
||||
}
|
||||
Source source = findWebhookSource(webhookId);
|
||||
if (source == null) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No such webhook");
|
||||
}
|
||||
|
||||
WebhookConfig config = WebhookConfig.from(source.options());
|
||||
byte[] body = readBoundedBody(request);
|
||||
if (!WebhookSignatures.verify(config.signingSecret(), body, signature)) {
|
||||
// Same 401 whether the header was absent or wrong: never confirm a guess.
|
||||
throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "Invalid signature");
|
||||
}
|
||||
if (!source.enabled()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.FORBIDDEN, "Webhook source is paused; deliveries are not accepted");
|
||||
}
|
||||
if (body.length == 0) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Empty request body");
|
||||
}
|
||||
|
||||
String storedName =
|
||||
config.usesConnection()
|
||||
? stageToConnection(config, filename, body)
|
||||
: stageToSpool(webhookId, filename, body);
|
||||
|
||||
// Fire the referencing policies now; the trigger's reconcile is the safety net.
|
||||
webhookTrigger.fireForWebhook(webhookId);
|
||||
log.info(
|
||||
"Accepted webhook delivery '{}' ({} bytes) for {}",
|
||||
storedName,
|
||||
body.length,
|
||||
webhookId);
|
||||
return ResponseEntity.accepted()
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.body(new WebhookDeliveryResponse(true, storedName, body.length));
|
||||
}
|
||||
|
||||
/** The enabled-or-not webhook source whose routing id matches, or null if there is none. */
|
||||
private Source findWebhookSource(String webhookId) {
|
||||
for (Source source : sourceStore.all()) {
|
||||
if (!WEBHOOK_TYPE.equals(source.type())) {
|
||||
continue;
|
||||
}
|
||||
Object configured = source.options().get(WebhookConfig.WEBHOOK_ID_OPTION);
|
||||
if (configured != null && configured.toString().equals(webhookId)) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Stage a delivery to the node-local spool; returns its display (original) name. */
|
||||
private String stageToSpool(String webhookId, String filename, byte[] body) {
|
||||
try {
|
||||
return WebhookSpool.displayName(
|
||||
spool.store(webhookId, filename, body).getFileName().toString());
|
||||
} catch (IOException e) {
|
||||
log.error("Could not spool webhook delivery for {}: {}", webhookId, e.getMessage());
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.INTERNAL_SERVER_ERROR, "Could not store delivery");
|
||||
}
|
||||
}
|
||||
|
||||
/** Stage a delivery to the S3 connection (save-time access check trusted). */
|
||||
private String stageToConnection(WebhookConfig config, String filename, byte[] body) {
|
||||
S3Config s3 =
|
||||
connectionResolver.resolve(
|
||||
Map.of(
|
||||
WebhookConfig.CONNECTION_ID_OPTION,
|
||||
config.connectionId(),
|
||||
"prefix",
|
||||
config.stagingPrefix()));
|
||||
S3Client client = connectionPool.clientFor(s3);
|
||||
String key = keyPrefix(s3.prefix()) + WebhookSpool.objectKeySuffix(filename);
|
||||
try {
|
||||
client.putObject(
|
||||
PutObjectRequest.builder().bucket(s3.bucket()).key(key).build(),
|
||||
RequestBody.fromBytes(body));
|
||||
} catch (SdkException e) {
|
||||
log.error(
|
||||
"Could not stage webhook delivery for {} to s3://{}/{}: {}",
|
||||
config.webhookId(),
|
||||
s3.bucket(),
|
||||
key,
|
||||
e.getMessage());
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.INTERNAL_SERVER_ERROR, "Could not store delivery");
|
||||
}
|
||||
return WebhookSpool.objectDisplayName(filename);
|
||||
}
|
||||
|
||||
/** The configured prefix as a key-path prefix ("inbox" and "inbox/" mean the same). */
|
||||
private static String keyPrefix(String prefix) {
|
||||
if (prefix == null || prefix.isEmpty() || prefix.endsWith("/")) {
|
||||
return prefix == null ? "" : prefix;
|
||||
}
|
||||
return prefix + "/";
|
||||
}
|
||||
|
||||
/** Read the body into memory, capped at {@code policies.webhookMaxBytes}. */
|
||||
private byte[] readBoundedBody(HttpServletRequest request) {
|
||||
long maxBytes = applicationProperties.getPolicies().getWebhookMaxBytes();
|
||||
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
|
||||
byte[] chunk = new byte[8192];
|
||||
long total = 0;
|
||||
try (InputStream in = request.getInputStream()) {
|
||||
int read;
|
||||
while ((read = in.read(chunk)) != -1) {
|
||||
total += read;
|
||||
if (total > maxBytes) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.PAYLOAD_TOO_LARGE,
|
||||
"Delivery exceeds the " + maxBytes + "-byte limit");
|
||||
}
|
||||
buffer.write(chunk, 0, read);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Could not read request body");
|
||||
}
|
||||
return buffer.toByteArray();
|
||||
}
|
||||
|
||||
/** The 202 body: the stored (display) name and byte count of an accepted delivery. */
|
||||
public record WebhookDeliveryResponse(boolean accepted, String filename, int bytes) {}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
/** HMAC-SHA256 signing of a webhook delivery's raw body (header sha256=<hex>). */
|
||||
public final class WebhookSignatures {
|
||||
|
||||
private static final String ALGORITHM = "HmacSHA256";
|
||||
private static final String PREFIX = "sha256=";
|
||||
|
||||
private WebhookSignatures() {}
|
||||
|
||||
/** The header a sender presents for {@code body}: {@code sha256=<lowercase hex>}. */
|
||||
public static String sign(String signingSecret, byte[] body) {
|
||||
return PREFIX + HexFormat.of().formatHex(hmac(signingSecret, body));
|
||||
}
|
||||
|
||||
/** Whether the presented signature is valid for {@code body}; false on bad input. */
|
||||
public static boolean verify(String signingSecret, byte[] body, String presented) {
|
||||
if (signingSecret == null || presented == null || body == null) {
|
||||
return false;
|
||||
}
|
||||
String hex = presented.trim();
|
||||
if (hex.regionMatches(true, 0, PREFIX, 0, PREFIX.length())) {
|
||||
hex = hex.substring(PREFIX.length());
|
||||
}
|
||||
byte[] presentedBytes;
|
||||
try {
|
||||
presentedBytes = HexFormat.of().parseHex(hex);
|
||||
} catch (IllegalArgumentException notHex) {
|
||||
return false;
|
||||
}
|
||||
return MessageDigest.isEqual(hmac(signingSecret, body), presentedBytes);
|
||||
}
|
||||
|
||||
private static byte[] hmac(String signingSecret, byte[] body) {
|
||||
try {
|
||||
Mac mac = Mac.getInstance(ALGORITHM);
|
||||
mac.init(new SecretKeySpec(signingSecret.getBytes(StandardCharsets.UTF_8), ALGORITHM));
|
||||
return mac.doFinal(body);
|
||||
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
|
||||
// HmacSHA256 is a required JCE algorithm and the key is always non-empty here.
|
||||
throw new IllegalStateException("HMAC-SHA256 unavailable", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.common.configuration.InstallationPathConfig;
|
||||
|
||||
/** Server-owned per-webhook staging directory under the install path. */
|
||||
@Component
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class WebhookSpool {
|
||||
|
||||
private static final String SPOOL_DIR = "policy-webhook-spool";
|
||||
private static final String TEMP_SUFFIX = ".part";
|
||||
private static final String DEFAULT_NAME = "document.pdf";
|
||||
private static final int UNIQUE_LEN = 32; // UUID hex without dashes
|
||||
|
||||
private final Path spoolRoot;
|
||||
|
||||
public WebhookSpool() {
|
||||
this(Path.of(InstallationPathConfig.getPath(), SPOOL_DIR));
|
||||
}
|
||||
|
||||
// Lets a caller (and tests) root the spool at a chosen directory; Spring uses the no-arg one.
|
||||
public WebhookSpool(Path spoolRoot) {
|
||||
this.spoolRoot = spoolRoot.toAbsolutePath().normalize();
|
||||
}
|
||||
|
||||
/** The staging directory for one webhook. Never escapes the spool root; may not yet exist. */
|
||||
public Path dirFor(String webhookId) {
|
||||
if (!WebhookIds.isValidId(webhookId)) {
|
||||
throw new IllegalArgumentException("invalid webhookId");
|
||||
}
|
||||
Path dir = spoolRoot.resolve(webhookId).normalize();
|
||||
if (!dir.getParent().equals(spoolRoot)) {
|
||||
// A validated id is a single safe segment; this only trips on a bug, never user input.
|
||||
throw new IllegalArgumentException("invalid webhookId");
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** Write a delivery into the spool atomically (staged .part, then moved into place). */
|
||||
public Path store(String webhookId, String filename, byte[] content) throws IOException {
|
||||
Path dir = dirFor(webhookId);
|
||||
Files.createDirectories(dir);
|
||||
String finalName = spoolName(filename);
|
||||
Path target = dir.resolve(finalName);
|
||||
Path temp = dir.resolve("." + finalName + TEMP_SUFFIX);
|
||||
Files.write(temp, content);
|
||||
try {
|
||||
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException atomicUnsupported) {
|
||||
Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
/** The spool file name for a delivery: a unique prefix plus the sanitised original name. */
|
||||
static String spoolName(String filename) {
|
||||
return UUID.randomUUID().toString().replace("-", "") + "-" + sanitize(filename);
|
||||
}
|
||||
|
||||
/** The original name recovered from a spool file name, for the resolved input's filename. */
|
||||
public static String displayName(String spoolFileName) {
|
||||
int dash = spoolFileName.indexOf('-');
|
||||
// The unique prefix is fixed-length hex with no dashes, so the first dash is the separator.
|
||||
if (dash == UNIQUE_LEN && dash + 1 < spoolFileName.length()) {
|
||||
return spoolFileName.substring(dash + 1);
|
||||
}
|
||||
return spoolFileName;
|
||||
}
|
||||
|
||||
/** S3 object-key suffix {@code <unique>/<name>}; the subfolder keeps the basename clean. */
|
||||
public static String objectKeySuffix(String filename) {
|
||||
return UUID.randomUUID().toString().replace("-", "") + "/" + sanitize(filename);
|
||||
}
|
||||
|
||||
/** The staged delivery's display name (basename of {@link #objectKeySuffix}). */
|
||||
public static String objectDisplayName(String filename) {
|
||||
return sanitize(filename);
|
||||
}
|
||||
|
||||
/** Reduce a client-supplied filename to a safe bare basename; fall back to a default. */
|
||||
private static String sanitize(String filename) {
|
||||
if (filename == null) {
|
||||
return DEFAULT_NAME;
|
||||
}
|
||||
String base = filename.replace('\\', '/');
|
||||
int slash = base.lastIndexOf('/');
|
||||
if (slash >= 0) {
|
||||
base = base.substring(slash + 1);
|
||||
}
|
||||
base = base.replaceAll("[^A-Za-z0-9._-]", "_").trim();
|
||||
// Strip leading dots so the result is never hidden (which resolve would skip) or empty.
|
||||
while (base.startsWith(".")) {
|
||||
base = base.substring(1);
|
||||
}
|
||||
return base.isEmpty() ? DEFAULT_NAME : base;
|
||||
}
|
||||
}
|
||||
@@ -20,9 +20,8 @@ public final class SecretMasker {
|
||||
private static final Pattern SENSITIVE =
|
||||
RegexPatternUtils.getInstance()
|
||||
.getPattern(
|
||||
// secret[_-]?access[_-]?key precedes plain secret so camelCase keys
|
||||
// like secretAccessKey (no word boundary after "secret") still match.
|
||||
"(?i)\\b(password|token|secret[_-]?access[_-]?key|secret|api[_-]?key|authorization|auth|jwt|cred|cert)\\b");
|
||||
// specific keys precede plain "secret" so camelCase still matches.
|
||||
"(?i)\\b(password|token|secret[_-]?access[_-]?key|signing[_-]?secret|secret|api[_-]?key|authorization|auth|jwt|cred|cert)\\b");
|
||||
|
||||
private SecretMasker() {}
|
||||
|
||||
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookConfig;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookSpool;
|
||||
|
||||
/** Tests for {@link WebhookInputSource}: ledger-backed read/consume, and id/secret minting. */
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class WebhookInputSourceTest {
|
||||
|
||||
private static final String POLICY = "p1";
|
||||
private static final String WEBHOOK_ID = "testwebhookid1234";
|
||||
|
||||
@Mock private FileReadinessChecker readinessChecker;
|
||||
@Mock private S3InputSource s3InputSource;
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private WebhookSpool spool;
|
||||
private WebhookInputSource source;
|
||||
private InProcessProcessedLedger ledger;
|
||||
private RecordingContext ctx;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
spool = new WebhookSpool(tempDir.resolve("spool"));
|
||||
source = new WebhookInputSource(spool, readinessChecker, s3InputSource);
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
lenient().when(readinessChecker.isReady(any())).thenReturn(true);
|
||||
}
|
||||
|
||||
private static InputSpec spec(String mode) {
|
||||
return new InputSpec(
|
||||
"webhook",
|
||||
Map.of("webhookId", WEBHOOK_ID, "signingSecret", "secret", "mode", mode));
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumeRemovesTheDeliveryOnceProcessed() throws IOException {
|
||||
Path delivered = spool.store(WEBHOOK_ID, "doc.pdf", "data".getBytes());
|
||||
|
||||
List<ResolvedInput> work = source.resolve(spec("consume"), ctx);
|
||||
|
||||
assertEquals(1, work.size());
|
||||
assertEquals("doc.pdf", work.get(0).inputs().primary().get(0).getFilename());
|
||||
// In flight: still spooled, but a second sweep does not pick it up again.
|
||||
assertTrue(Files.exists(delivered));
|
||||
assertTrue(source.resolve(spec("consume"), ctx).isEmpty());
|
||||
|
||||
work.get(0).onComplete().accept(true);
|
||||
assertTrue(Files.notExists(delivered));
|
||||
assertTrue(source.resolve(spec("consume"), ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedRunLeavesTheDeliveryInPlace() throws IOException {
|
||||
Path delivered = spool.store(WEBHOOK_ID, "doc.pdf", "data".getBytes());
|
||||
|
||||
List<ResolvedInput> work = source.resolve(spec("consume"), ctx);
|
||||
work.get(0).onComplete().accept(false);
|
||||
|
||||
assertTrue(Files.exists(delivered));
|
||||
}
|
||||
|
||||
@Test
|
||||
void snapshotReReadsEveryRunAndNeverDeletes() throws IOException {
|
||||
Path delivered = spool.store(WEBHOOK_ID, "doc.pdf", "data".getBytes());
|
||||
|
||||
assertEquals(1, source.resolve(spec("snapshot"), ctx).size());
|
||||
List<ResolvedInput> second = source.resolve(spec("snapshot"), ctx);
|
||||
assertEquals(1, second.size());
|
||||
second.get(0).onComplete().accept(true);
|
||||
assertTrue(Files.exists(delivered));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nothingDeliveredIsAnEmptySourceNotAnError() throws IOException {
|
||||
List<ResolvedInput> work = source.resolve(spec("consume"), ctx);
|
||||
assertTrue(work.isEmpty());
|
||||
assertTrue(ctx.present.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsMissingIdOrSecret() {
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> source.validate(new InputSpec("webhook", Map.of("signingSecret", "s"))));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> source.validate(new InputSpec("webhook", Map.of("webhookId", WEBHOOK_ID))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void prepareMintsIdAndSecretOnCreate() {
|
||||
Map<String, Object> prepared =
|
||||
source.prepareOptionsForSave(Map.of("mode", "consume"), true);
|
||||
|
||||
String id = prepared.get(WebhookConfig.WEBHOOK_ID_OPTION).toString();
|
||||
String secret = prepared.get(WebhookConfig.SIGNING_SECRET_OPTION).toString();
|
||||
assertFalse(id.isBlank());
|
||||
assertFalse(secret.isBlank());
|
||||
assertEquals("consume", prepared.get("mode"));
|
||||
Map<String, Object> other = source.prepareOptionsForSave(Map.of(), true);
|
||||
assertNotEquals(id, other.get(WebhookConfig.WEBHOOK_ID_OPTION).toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void prepareLeavesAnExistingWebhookUntouchedOnEdit() {
|
||||
Map<String, Object> existing =
|
||||
Map.of("webhookId", WEBHOOK_ID, "signingSecret", "keepme", "mode", "snapshot");
|
||||
|
||||
Map<String, Object> prepared = source.prepareOptionsForSave(existing, false);
|
||||
|
||||
assertEquals(WEBHOOK_ID, prepared.get("webhookId"));
|
||||
assertEquals("keepme", prepared.get("signingSecret"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aConnectionBackedWebhookDelegatesToTheS3SourceUnderItsReservedPrefix() throws IOException {
|
||||
when(s3InputSource.resolve(any(), any())).thenReturn(List.of());
|
||||
InputSpec spec =
|
||||
new InputSpec(
|
||||
"webhook",
|
||||
Map.of(
|
||||
"webhookId",
|
||||
WEBHOOK_ID,
|
||||
"signingSecret",
|
||||
"secret",
|
||||
"mode",
|
||||
"consume",
|
||||
"connectionId",
|
||||
7));
|
||||
|
||||
source.resolve(spec, ctx);
|
||||
|
||||
ArgumentCaptor<InputSpec> delegated = ArgumentCaptor.forClass(InputSpec.class);
|
||||
verify(s3InputSource).resolve(delegated.capture(), eq(ctx));
|
||||
InputSpec s3 = delegated.getValue();
|
||||
assertEquals("s3", s3.type());
|
||||
assertEquals(7L, ((Number) s3.options().get("connectionId")).longValue());
|
||||
assertEquals("stirling-webhook/" + WEBHOOK_ID, s3.options().get("prefix"));
|
||||
assertEquals("consume", s3.options().get("mode"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aConnectionBackedWebhookValidatesTheConnectionThroughTheS3Source() {
|
||||
InputSpec spec =
|
||||
new InputSpec(
|
||||
"webhook",
|
||||
Map.of(
|
||||
"webhookId",
|
||||
WEBHOOK_ID,
|
||||
"signingSecret",
|
||||
"secret",
|
||||
"connectionId",
|
||||
7));
|
||||
|
||||
source.validate(spec);
|
||||
|
||||
// Save-time validation defers to the S3 source, which ownership-checks the connection.
|
||||
verify(s3InputSource).validate(any());
|
||||
}
|
||||
|
||||
/** Policy-scoped context backed by the in-process ledger, recording presence reports. */
|
||||
private class RecordingContext implements ResolveContext {
|
||||
|
||||
private final List<String> present = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public boolean claim(String identity, String gate, Supplier<String> contentHash) {
|
||||
return ledger.claim(POLICY, identity, gate, contentHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(
|
||||
String identity, String finalGate, String finalContentHash, boolean success) {
|
||||
ledger.settle(POLICY, identity, finalGate, finalContentHash, success);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allSettledDone(String identity) {
|
||||
return ledger.allSettledDone(identity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reportPresent(Collection<String> identities) {
|
||||
present.addAll(identities);
|
||||
}
|
||||
}
|
||||
}
|
||||
+57
-1
@@ -1,33 +1,42 @@
|
||||
package stirling.software.proprietary.policy.source;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.S3InputSource;
|
||||
import stirling.software.proprietary.policy.input.WebhookInputSource;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookSpool;
|
||||
import stirling.software.proprietary.util.SecretMasker;
|
||||
|
||||
/**
|
||||
@@ -41,6 +50,9 @@ class SourceControllerTest {
|
||||
private final PolicyStore policyStore = new InProcessPolicyStore();
|
||||
private PolicyTriggerManager triggerManager;
|
||||
private SourceController controller;
|
||||
private SourceController webhookController;
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
@@ -58,9 +70,11 @@ class SourceControllerTest {
|
||||
policyGuard,
|
||||
new InProcessSourceDocCounter());
|
||||
triggerManager = mock(PolicyTriggerManager.class);
|
||||
// A permissive input source so config validation passes and save can be exercised.
|
||||
// A permissive input source; stub prepareOptionsForSave as a pass-through.
|
||||
InputSource folderInput = mock(InputSource.class);
|
||||
when(folderInput.supports(any())).thenReturn(true);
|
||||
when(folderInput.prepareOptionsForSave(any(), anyBoolean()))
|
||||
.thenAnswer(invocation -> invocation.getArgument(0));
|
||||
controller =
|
||||
new SourceController(
|
||||
sourceStore,
|
||||
@@ -72,6 +86,48 @@ class SourceControllerTest {
|
||||
triggerManager,
|
||||
properties,
|
||||
List.of(folderInput));
|
||||
WebhookInputSource webhookInput =
|
||||
new WebhookInputSource(
|
||||
new WebhookSpool(tempDir),
|
||||
mock(FileReadinessChecker.class),
|
||||
mock(S3InputSource.class));
|
||||
webhookController =
|
||||
new SourceController(
|
||||
sourceStore,
|
||||
sourceGuard,
|
||||
overviewService,
|
||||
policyStore,
|
||||
policyGuard,
|
||||
authority,
|
||||
triggerManager,
|
||||
properties,
|
||||
List.of(webhookInput));
|
||||
}
|
||||
|
||||
@Test
|
||||
void creatingAWebhookRevealsItsSecretOnceThenMasks() {
|
||||
Source created =
|
||||
webhookController
|
||||
.save(
|
||||
new Source(
|
||||
null,
|
||||
"Partner uploads",
|
||||
"webhook",
|
||||
Map.of("mode", "consume"),
|
||||
true,
|
||||
null,
|
||||
null))
|
||||
.getBody();
|
||||
|
||||
String secret = String.valueOf(created.options().get("signingSecret"));
|
||||
String webhookId = String.valueOf(created.options().get("webhookId"));
|
||||
assertNotEquals(SecretMasker.REDACTED, secret);
|
||||
assertFalse(secret.isBlank());
|
||||
assertFalse(webhookId.isBlank());
|
||||
|
||||
Source read = webhookController.get(created.id()).getBody();
|
||||
assertEquals(SecretMasker.REDACTED, read.options().get("signingSecret"));
|
||||
assertEquals(webhookId, read.options().get("webhookId"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package stirling.software.proprietary.policy.trigger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.TriggerConfig;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
|
||||
/** Tests for {@link WebhookTrigger}: fires only policies referencing the delivered-to webhook. */
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class WebhookTriggerTest {
|
||||
|
||||
private static final String TYPE = "webhook";
|
||||
|
||||
@Mock private PolicyStore policyStore;
|
||||
@Mock private PolicyRunner policyRunner;
|
||||
|
||||
private final SourceStore sourceStore = new InProcessSourceStore();
|
||||
private WebhookTrigger trigger;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
trigger =
|
||||
new WebhookTrigger(
|
||||
policyStore, policyRunner, sourceStore, new ApplicationProperties());
|
||||
}
|
||||
|
||||
@Test
|
||||
void firesOnlyPoliciesReferencingTheDeliveredWebhook() {
|
||||
Policy matching = webhookPolicy("a", "whkA");
|
||||
Policy other = webhookPolicy("b", "whkB");
|
||||
when(policyStore.findByTriggerType(TYPE)).thenReturn(List.of(matching, other));
|
||||
|
||||
trigger.fireForWebhook("whkA");
|
||||
|
||||
verify(policyRunner).run(matching, SweepKind.LIGHT);
|
||||
verify(policyRunner, never()).run(other, SweepKind.LIGHT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoresADeliveryForAnUnknownWebhookId() {
|
||||
Policy policy = webhookPolicy("a", "whkA");
|
||||
when(policyStore.findByTriggerType(TYPE)).thenReturn(List.of(policy));
|
||||
|
||||
trigger.fireForWebhook("whkZ");
|
||||
|
||||
verify(policyRunner, never()).run(any(), any(SweepKind.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRequiresAWebhookSource() {
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> trigger.validate(policy("p", webhookTriggerConfig(), List.of())));
|
||||
trigger.validate(webhookPolicy("p", "whkA"));
|
||||
}
|
||||
|
||||
private static TriggerConfig webhookTriggerConfig() {
|
||||
return new TriggerConfig(TYPE, Map.of());
|
||||
}
|
||||
|
||||
/** Persist a webhook source with the given routing id and return a policy referencing it. */
|
||||
private Policy webhookPolicy(String id, String webhookId) {
|
||||
String sourceId =
|
||||
sourceStore
|
||||
.save(
|
||||
new Source(
|
||||
null,
|
||||
"hook",
|
||||
"webhook",
|
||||
Map.of(
|
||||
"webhookId",
|
||||
webhookId,
|
||||
"signingSecret",
|
||||
"s",
|
||||
"mode",
|
||||
"consume"),
|
||||
true,
|
||||
"owner",
|
||||
null))
|
||||
.id();
|
||||
return policy(id, webhookTriggerConfig(), List.of(sourceId));
|
||||
}
|
||||
|
||||
private static Policy policy(String id, TriggerConfig trigger, List<String> sourceIds) {
|
||||
return new Policy(
|
||||
id,
|
||||
"hook",
|
||||
"owner",
|
||||
true,
|
||||
trigger,
|
||||
sourceIds,
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
OutputSpec.inline());
|
||||
}
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockHttpServletRequest;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.trigger.WebhookTrigger;
|
||||
import stirling.software.proprietary.policy.webhook.WebhookReceiverController.WebhookDeliveryResponse;
|
||||
|
||||
/** Tests for the public webhook receiver: valid delivery spools + fires; bad requests rejected. */
|
||||
class WebhookReceiverControllerTest {
|
||||
|
||||
private static final String WEBHOOK_ID = "receivertestid12";
|
||||
private static final String SECRET = "topsecret";
|
||||
private static final byte[] BODY = "a pdf".getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private SourceStore sourceStore;
|
||||
private WebhookSpool spool;
|
||||
private WebhookTrigger trigger;
|
||||
private ApplicationProperties properties;
|
||||
private WebhookReceiverController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
sourceStore = new InProcessSourceStore();
|
||||
sourceStore.save(webhookSource(true));
|
||||
spool = new WebhookSpool(tempDir.resolve("spool"));
|
||||
trigger = mock(WebhookTrigger.class);
|
||||
properties = new ApplicationProperties();
|
||||
// The local-disk tests never touch a connection; mocks stand in for the S3 collaborators.
|
||||
controller =
|
||||
new WebhookReceiverController(
|
||||
sourceStore,
|
||||
spool,
|
||||
trigger,
|
||||
properties,
|
||||
mock(S3ConnectionResolver.class),
|
||||
mock(S3ConnectionPool.class));
|
||||
}
|
||||
|
||||
private static Source webhookSource(boolean enabled) {
|
||||
return new Source(
|
||||
"s1",
|
||||
"Partner uploads",
|
||||
"webhook",
|
||||
Map.of("webhookId", WEBHOOK_ID, "signingSecret", SECRET, "mode", "consume"),
|
||||
enabled,
|
||||
"owner",
|
||||
null);
|
||||
}
|
||||
|
||||
private static MockHttpServletRequest request(byte[] body) {
|
||||
MockHttpServletRequest req =
|
||||
new MockHttpServletRequest("POST", "/api/v1/webhooks/" + WEBHOOK_ID);
|
||||
req.setContent(body);
|
||||
return req;
|
||||
}
|
||||
|
||||
@Test
|
||||
void aValidDeliveryIsSpooledAndFiresTheTrigger() throws IOException {
|
||||
String signature = WebhookSignatures.sign(SECRET, BODY);
|
||||
|
||||
ResponseEntity<WebhookDeliveryResponse> response =
|
||||
controller.receive(WEBHOOK_ID, signature, "invoice.pdf", request(BODY));
|
||||
|
||||
assertEquals(202, response.getStatusCode().value());
|
||||
assertTrue(response.getBody().accepted());
|
||||
assertEquals("invoice.pdf", response.getBody().filename());
|
||||
assertEquals(1, spooledFiles().size());
|
||||
verify(trigger).fireForWebhook(WEBHOOK_ID);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aWrongSignatureIsRejectedAndStoresNothing() {
|
||||
ResponseStatusException ex =
|
||||
assertThrows(
|
||||
ResponseStatusException.class,
|
||||
() ->
|
||||
controller.receive(
|
||||
WEBHOOK_ID, "sha256=deadbeef", "x.pdf", request(BODY)));
|
||||
|
||||
assertEquals(401, ex.getStatusCode().value());
|
||||
assertTrue(spooledFiles().isEmpty());
|
||||
verify(trigger, never()).fireForWebhook(WEBHOOK_ID);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anUnknownWebhookIsNotFound() {
|
||||
ResponseStatusException ex =
|
||||
assertThrows(
|
||||
ResponseStatusException.class,
|
||||
() ->
|
||||
controller.receive(
|
||||
"unknownwebhookid",
|
||||
WebhookSignatures.sign(SECRET, BODY),
|
||||
"x.pdf",
|
||||
request(BODY)));
|
||||
|
||||
assertEquals(404, ex.getStatusCode().value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aPausedSourceRejectsDeliveries() {
|
||||
sourceStore.save(webhookSource(false));
|
||||
String signature = WebhookSignatures.sign(SECRET, BODY);
|
||||
|
||||
ResponseStatusException ex =
|
||||
assertThrows(
|
||||
ResponseStatusException.class,
|
||||
() -> controller.receive(WEBHOOK_ID, signature, "x.pdf", request(BODY)));
|
||||
|
||||
assertEquals(403, ex.getStatusCode().value());
|
||||
assertTrue(spooledFiles().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void anEmptyBodyIsRejected() {
|
||||
byte[] empty = new byte[0];
|
||||
String signature = WebhookSignatures.sign(SECRET, empty);
|
||||
|
||||
ResponseStatusException ex =
|
||||
assertThrows(
|
||||
ResponseStatusException.class,
|
||||
() -> controller.receive(WEBHOOK_ID, signature, null, request(empty)));
|
||||
|
||||
assertEquals(400, ex.getStatusCode().value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void anOversizeDeliveryIsRejectedBeforeStoring() {
|
||||
properties.getPolicies().setWebhookMaxBytes(2);
|
||||
|
||||
ResponseStatusException ex =
|
||||
assertThrows(
|
||||
ResponseStatusException.class,
|
||||
() ->
|
||||
controller.receive(
|
||||
WEBHOOK_ID,
|
||||
WebhookSignatures.sign(SECRET, BODY),
|
||||
"x.pdf",
|
||||
request(BODY)));
|
||||
|
||||
assertEquals(413, ex.getStatusCode().value());
|
||||
assertTrue(spooledFiles().isEmpty());
|
||||
}
|
||||
|
||||
private List<Path> spooledFiles() {
|
||||
Path dir = spool.dirFor(WEBHOOK_ID);
|
||||
if (!Files.isDirectory(dir)) {
|
||||
return List.of();
|
||||
}
|
||||
try (Stream<Path> entries = Files.list(dir)) {
|
||||
return entries.filter(Files::isRegularFile)
|
||||
.filter(p -> !p.getFileName().toString().startsWith("."))
|
||||
.toList();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.mock.web.MockHttpServletRequest;
|
||||
import org.testcontainers.containers.MinIOContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.policy.input.ResolveContext;
|
||||
import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
import stirling.software.proprietary.policy.input.S3InputSource;
|
||||
import stirling.software.proprietary.policy.input.WebhookInputSource;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.trigger.WebhookTrigger;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.S3Configuration;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/** End-to-end connection-backed webhook against MinIO: stage, read via S3 delegation, consume. */
|
||||
@Testcontainers(disabledWithoutDocker = true)
|
||||
class WebhookS3ConnectionMinioTest {
|
||||
|
||||
private static final String POLICY = "p1";
|
||||
private static final String ACCESS_KEY = "minioadmin";
|
||||
private static final String SECRET_KEY = "minioadmin";
|
||||
private static final String WEBHOOK_ID = "miniowebhookid12";
|
||||
private static final String SECRET = "topsecret";
|
||||
private static final long CONNECTION_ID = 7L;
|
||||
|
||||
@Container
|
||||
static MinIOContainer minio =
|
||||
new MinIOContainer("minio/minio:latest")
|
||||
.withUserName(ACCESS_KEY)
|
||||
.withPassword(SECRET_KEY);
|
||||
|
||||
private static S3Client adminClient;
|
||||
private static int bucketCounter;
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private String bucket;
|
||||
private String stagingPrefix;
|
||||
private WebhookReceiverController receiver;
|
||||
private WebhookInputSource inputSource;
|
||||
private InProcessProcessedLedger ledger;
|
||||
private RecordingContext ctx;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
if (adminClient == null) {
|
||||
adminClient =
|
||||
S3Client.builder()
|
||||
.endpointOverride(URI.create(minio.getS3URL()))
|
||||
.httpClient(UrlConnectionHttpClient.create())
|
||||
.region(Region.US_EAST_1)
|
||||
.credentialsProvider(
|
||||
StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
|
||||
.serviceConfiguration(
|
||||
S3Configuration.builder().pathStyleAccessEnabled(true).build())
|
||||
.build();
|
||||
}
|
||||
bucket = "webhook-inbox-" + ++bucketCounter;
|
||||
adminClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
|
||||
stagingPrefix = "stirling-webhook/" + WEBHOOK_ID + "/";
|
||||
|
||||
// The MinIO endpoint resolves to loopback, so the operator opt-in must be on.
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(true);
|
||||
|
||||
// A resolver over a MinIO S3 connection; no principal here, so ownership is skipped.
|
||||
S3ConnectionResolver resolver = resolverFor(minioConnection());
|
||||
S3ConnectionPool pool = new S3ConnectionPool(properties);
|
||||
S3InputSource s3 = new S3InputSource(pool, resolver);
|
||||
|
||||
inputSource =
|
||||
new WebhookInputSource(
|
||||
new WebhookSpool(tempDir.resolve("spool")),
|
||||
mock(FileReadinessChecker.class),
|
||||
s3);
|
||||
|
||||
InProcessSourceStore sourceStore = new InProcessSourceStore();
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
"s1",
|
||||
"Partner uploads",
|
||||
"webhook",
|
||||
Map.of(
|
||||
"webhookId", WEBHOOK_ID,
|
||||
"signingSecret", SECRET,
|
||||
"mode", "consume",
|
||||
"connectionId", CONNECTION_ID),
|
||||
true,
|
||||
"owner",
|
||||
null));
|
||||
receiver =
|
||||
new WebhookReceiverController(
|
||||
sourceStore,
|
||||
new WebhookSpool(tempDir.resolve("spool")),
|
||||
mock(WebhookTrigger.class),
|
||||
properties,
|
||||
resolver,
|
||||
pool);
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDeliveryIsStagedToTheConnectionThenReadAndConsumed() throws IOException {
|
||||
byte[] body = "a pdf".getBytes(StandardCharsets.UTF_8);
|
||||
String signature = WebhookSignatures.sign(SECRET, body);
|
||||
|
||||
var response = receiver.receive(WEBHOOK_ID, signature, "invoice.pdf", request(body));
|
||||
assertThat(response.getStatusCode().value()).isEqualTo(202);
|
||||
|
||||
List<S3Object> staged = listUnder(stagingPrefix);
|
||||
assertThat(staged).hasSize(1);
|
||||
// Staged under a unique subfolder so the object basename stays the clean original name.
|
||||
assertThat(staged.get(0).key()).endsWith("/invoice.pdf");
|
||||
|
||||
List<ResolvedInput> work = inputSource.resolve(webhookSpec(), ctx);
|
||||
assertThat(work).hasSize(1);
|
||||
assertThat(work.get(0).inputs().primary().get(0).getFilename()).isEqualTo("invoice.pdf");
|
||||
assertThat(read(work.get(0))).isEqualTo("a pdf");
|
||||
// In flight: a second sweep claims nothing.
|
||||
assertThat(inputSource.resolve(webhookSpec(), ctx)).isEmpty();
|
||||
|
||||
work.get(0).onComplete().accept(true);
|
||||
assertThat(listUnder(stagingPrefix)).isEmpty();
|
||||
assertThat(inputSource.resolve(webhookSpec(), ctx)).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRejectedDeliveryStagesNothing() {
|
||||
byte[] body = "a pdf".getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
try {
|
||||
receiver.receive(WEBHOOK_ID, "sha256=deadbeef", "x.pdf", request(body));
|
||||
} catch (RuntimeException expected) {
|
||||
// 401 invalid signature
|
||||
}
|
||||
|
||||
assertThat(listUnder(stagingPrefix)).isEmpty();
|
||||
}
|
||||
|
||||
private IntegrationConfig minioConnection() {
|
||||
IntegrationConfig connection = new IntegrationConfig();
|
||||
connection.setId(CONNECTION_ID);
|
||||
connection.setIntegrationType(IntegrationType.S3);
|
||||
connection.setName("minio");
|
||||
connection.setEnabled(true);
|
||||
connection.setConfig(
|
||||
new ObjectMapper()
|
||||
.writeValueAsString(
|
||||
Map.of(
|
||||
"bucket", bucket,
|
||||
"region", "us-east-1",
|
||||
"endpoint", minio.getS3URL(),
|
||||
"accessKeyId", ACCESS_KEY,
|
||||
"secretAccessKey", SECRET_KEY)));
|
||||
return connection;
|
||||
}
|
||||
|
||||
private static S3ConnectionResolver resolverFor(IntegrationConfig connection) {
|
||||
IntegrationConfigRepository connections = mock(IntegrationConfigRepository.class);
|
||||
when(connections.findById(connection.getId())).thenReturn(Optional.of(connection));
|
||||
return new S3ConnectionResolver(
|
||||
connections, mock(OwnershipService.class), mock(UserService.class));
|
||||
}
|
||||
|
||||
private static InputSpec webhookSpec() {
|
||||
return new InputSpec(
|
||||
"webhook",
|
||||
Map.of(
|
||||
"webhookId", WEBHOOK_ID,
|
||||
"signingSecret", SECRET,
|
||||
"mode", "consume",
|
||||
"connectionId", CONNECTION_ID));
|
||||
}
|
||||
|
||||
private static MockHttpServletRequest request(byte[] body) {
|
||||
MockHttpServletRequest req =
|
||||
new MockHttpServletRequest("POST", "/api/v1/webhooks/" + WEBHOOK_ID);
|
||||
req.setContent(body);
|
||||
return req;
|
||||
}
|
||||
|
||||
private List<S3Object> listUnder(String prefix) {
|
||||
return adminClient
|
||||
.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).prefix(prefix).build())
|
||||
.contents();
|
||||
}
|
||||
|
||||
private static String read(ResolvedInput unit) throws IOException {
|
||||
try (InputStream stream = unit.inputs().primary().get(0).getInputStream()) {
|
||||
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
private class RecordingContext implements ResolveContext {
|
||||
|
||||
private final List<String> present = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public boolean claim(String identity, String gate, Supplier<String> contentHash) {
|
||||
return ledger.claim(POLICY, identity, gate, contentHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(
|
||||
String identity, String finalGate, String finalContentHash, boolean success) {
|
||||
ledger.settle(POLICY, identity, finalGate, finalContentHash, success);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allSettledDone(String identity) {
|
||||
return ledger.allSettledDone(identity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reportPresent(Collection<String> identities) {
|
||||
present.addAll(identities);
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package stirling.software.proprietary.policy.webhook;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** The HMAC-SHA256 webhook signing scheme: correct signatures verify, tampered ones do not. */
|
||||
class WebhookSignaturesTest {
|
||||
|
||||
private static final String SECRET = "whsec_test_secret";
|
||||
private static final byte[] BODY = "the document bytes".getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
@Test
|
||||
void aFreshlySignedBodyVerifies() {
|
||||
String header = WebhookSignatures.sign(SECRET, BODY);
|
||||
assertTrue(header.startsWith("sha256="));
|
||||
assertTrue(WebhookSignatures.verify(SECRET, BODY, header));
|
||||
}
|
||||
|
||||
@Test
|
||||
void abarehexSignatureVerifiesToo() {
|
||||
String header = WebhookSignatures.sign(SECRET, BODY);
|
||||
String bareHex = header.substring("sha256=".length());
|
||||
assertTrue(WebhookSignatures.verify(SECRET, BODY, bareHex));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aWrongSecretDoesNotVerify() {
|
||||
String header = WebhookSignatures.sign(SECRET, BODY);
|
||||
assertFalse(WebhookSignatures.verify("other-secret", BODY, header));
|
||||
}
|
||||
|
||||
@Test
|
||||
void atamperedBodyDoesNotVerify() {
|
||||
String header = WebhookSignatures.sign(SECRET, BODY);
|
||||
byte[] tampered = "the document byteS".getBytes(StandardCharsets.UTF_8);
|
||||
assertFalse(WebhookSignatures.verify(SECRET, tampered, header));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMissingOrMalformedHeaderIsFalseNotAnError() {
|
||||
assertFalse(WebhookSignatures.verify(SECRET, BODY, null));
|
||||
assertFalse(WebhookSignatures.verify(SECRET, BODY, "sha256=not-hex"));
|
||||
assertFalse(WebhookSignatures.verify(SECRET, BODY, ""));
|
||||
}
|
||||
}
|
||||
+12
@@ -103,6 +103,18 @@ class SecretMaskerTest {
|
||||
assertEquals("AKIAEXAMPLE", result.get("accessKeyId"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should mask camelCase signingSecret despite no word boundary")
|
||||
void shouldMaskCamelCaseSigningSecret() {
|
||||
Map<String, Object> input = Map.of("signingSecret", "shh", "webhookId", "whk_abc");
|
||||
|
||||
Map<String, Object> result = SecretMasker.mask(input);
|
||||
|
||||
assertEquals(SecretMasker.REDACTED, result.get("signingSecret"));
|
||||
// The routing id is a public URL token, not a secret.
|
||||
assertEquals("whk_abc", result.get("webhookId"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should mask nested map sensitive keys")
|
||||
void shouldMaskNestedMapSensitiveKeys() {
|
||||
|
||||
@@ -8139,6 +8139,36 @@ placeholder = "us-east-1"
|
||||
[portal.sources.types.s3.fields.secretAccessKey]
|
||||
label = "Secret access key"
|
||||
|
||||
[portal.sources.types.webhook]
|
||||
description = "Receive documents by signed HTTP POST from any external system."
|
||||
label = "Webhook"
|
||||
|
||||
[portal.sources.types.webhook.detail]
|
||||
deliveryUrl = "Delivery URL"
|
||||
secretNote = "The signing secret is shown once, when the webhook is created. Recreate the source to roll it."
|
||||
|
||||
[portal.sources.types.webhook.fields.connection]
|
||||
helperText = "Choose an S3 connection to stage deliveries durably (required in hosted). Leave blank on a self-hosted server to stage them on local disk."
|
||||
label = "Storage connection"
|
||||
|
||||
[portal.sources.types.webhook.fields.mode]
|
||||
helperText = "Consume removes each delivered document once every policy has processed it."
|
||||
label = "Read mode"
|
||||
|
||||
[portal.sources.types.webhook.fields.mode.options]
|
||||
consume = "Consume: process each delivery once"
|
||||
snapshot = "Snapshot: re-read the spool every run"
|
||||
|
||||
[portal.sources.types.webhook.reveal]
|
||||
copy = "Copy"
|
||||
done = "Done"
|
||||
secret = "Signing secret"
|
||||
secretHelp = "Sign each delivery's raw body with this key (HMAC-SHA256) and send it as the X-Stirling-Signature header."
|
||||
secretWarning = "Copy the signing secret now. For your security it is shown only once and cannot be retrieved later."
|
||||
title = "Webhook created"
|
||||
url = "Delivery URL"
|
||||
usage = "POST each document as the raw request body to the delivery URL with a binary content type (application/pdf or application/octet-stream). Referencing policies run automatically on arrival."
|
||||
|
||||
[portal.sources.types.unknown]
|
||||
label = "Source"
|
||||
|
||||
|
||||
@@ -7,6 +7,15 @@ describe("creatableSourceTypes (SaaS)", () => {
|
||||
expect(creatableSourceTypes().map((t) => t.type)).not.toContain("folder");
|
||||
});
|
||||
|
||||
it("offers webhook with a required S3 connection (durable staging in hosted)", () => {
|
||||
const webhook = creatableSourceTypes().find((t) => t.type === "webhook");
|
||||
expect(webhook).toBeDefined();
|
||||
const connection = webhook?.fields.find(
|
||||
(f) => f.control === "s3Connection",
|
||||
);
|
||||
expect(connection?.required).toBe(true);
|
||||
});
|
||||
|
||||
it("still offers the cloud source types", () => {
|
||||
expect(creatableSourceTypes().map((t) => t.type)).toContain("s3");
|
||||
});
|
||||
|
||||
@@ -1,13 +1,25 @@
|
||||
import {
|
||||
CREATABLE_SOURCE_TYPES,
|
||||
WEBHOOK_SOURCE_TYPE,
|
||||
type CreatableSourceType,
|
||||
} from "@portal/components/sources/sourceTypes";
|
||||
|
||||
/**
|
||||
* Hosted deployments never read the server's filesystem (the backend's
|
||||
* FolderAccessGuard denies it outright), so folder connections are not offered
|
||||
* in the connect wizard.
|
||||
*/
|
||||
/** Hosted drops folder (no server FS) and requires webhook's S3 connection (node-local isn't durable). */
|
||||
export function creatableSourceTypes(): CreatableSourceType[] {
|
||||
return CREATABLE_SOURCE_TYPES.filter((type) => type.type !== "folder");
|
||||
return CREATABLE_SOURCE_TYPES.filter((type) => type.type !== "folder").map(
|
||||
(type) =>
|
||||
type.type === WEBHOOK_SOURCE_TYPE ? withRequiredConnection(type) : type,
|
||||
);
|
||||
}
|
||||
|
||||
/** The type with its S3-connection field marked required (durable staging is mandatory). */
|
||||
function withRequiredConnection(
|
||||
type: CreatableSourceType,
|
||||
): CreatableSourceType {
|
||||
return {
|
||||
...type,
|
||||
fields: type.fields.map((field) =>
|
||||
field.control === "s3Connection" ? { ...field, required: true } : field,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ export interface SourceView {
|
||||
docsTotal: number;
|
||||
docs24h: number;
|
||||
docs30d: number;
|
||||
/** Webhook sources only: the server-relative delivery path senders POST to. Null otherwise. */
|
||||
webhookPath?: string | null;
|
||||
}
|
||||
|
||||
export interface SourceKpi {
|
||||
|
||||
@@ -22,6 +22,9 @@ export interface SourceTypeMeta {
|
||||
*/
|
||||
export const EDITOR_SOURCE_TYPE = "editor";
|
||||
|
||||
/** The webhook source type. Its delivery URL + signing secret are minted server-side on create. */
|
||||
export const WEBHOOK_SOURCE_TYPE = "webhook";
|
||||
|
||||
const SOURCE_TYPE_META: Record<string, SourceTypeMeta> = {
|
||||
folder: {
|
||||
labelKey: "portal.sources.types.folder.label",
|
||||
@@ -38,6 +41,11 @@ const SOURCE_TYPE_META: Record<string, SourceTypeMeta> = {
|
||||
icon: "☁",
|
||||
accent: "brand",
|
||||
},
|
||||
webhook: {
|
||||
labelKey: "portal.sources.types.webhook.label",
|
||||
icon: "↯",
|
||||
accent: "warning",
|
||||
},
|
||||
};
|
||||
|
||||
const UNKNOWN_TYPE_META: SourceTypeMeta = {
|
||||
@@ -180,6 +188,40 @@ export const CREATABLE_SOURCE_TYPES: CreatableSourceType[] = [
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// URL + secret minted server-side (revealed once); an S3 connection makes staging durable, else local disk.
|
||||
type: WEBHOOK_SOURCE_TYPE,
|
||||
labelKey: "portal.sources.types.webhook.label",
|
||||
descriptionKey: "portal.sources.types.webhook.description",
|
||||
fields: [
|
||||
{
|
||||
key: "connectionId",
|
||||
labelKey: "portal.sources.types.webhook.fields.connection.label",
|
||||
control: "s3Connection",
|
||||
helperTextKey:
|
||||
"portal.sources.types.webhook.fields.connection.helperText",
|
||||
},
|
||||
{
|
||||
key: "mode",
|
||||
labelKey: "portal.sources.types.webhook.fields.mode.label",
|
||||
control: "select",
|
||||
defaultValue: "consume",
|
||||
helperTextKey: "portal.sources.types.webhook.fields.mode.helperText",
|
||||
options: [
|
||||
{
|
||||
value: "consume",
|
||||
labelKey:
|
||||
"portal.sources.types.webhook.fields.mode.options.consume",
|
||||
},
|
||||
{
|
||||
value: "snapshot",
|
||||
labelKey:
|
||||
"portal.sources.types.webhook.fields.mode.options.snapshot",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
/** Default option values for a type's create form. */
|
||||
|
||||
@@ -55,6 +55,18 @@ function seedSources(): StoredSource[] {
|
||||
enabled: false,
|
||||
owner: "data-eng@acme.com",
|
||||
},
|
||||
{
|
||||
id: "src-webhook",
|
||||
name: "Partner uploads",
|
||||
type: "webhook",
|
||||
options: {
|
||||
webhookId: "whk_demo_5f3a9c21b7",
|
||||
signingSecret: "whsec_demo_2b8e1d47a9f60c35",
|
||||
mode: "consume",
|
||||
},
|
||||
enabled: true,
|
||||
owner: "you@acme.com",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@@ -76,6 +88,7 @@ const docCounts: Record<
|
||||
"src-contracts": { total: 12840, last24h: 96, last30d: 2310 },
|
||||
"src-archive": { total: 1180, last24h: 0, last30d: 0 },
|
||||
"src-legacy": { total: 48600, last24h: 0, last30d: 0 },
|
||||
"src-webhook": { total: 3120, last24h: 24, last30d: 640 },
|
||||
};
|
||||
|
||||
function docsFor(id: string): {
|
||||
@@ -96,14 +109,23 @@ function nextId(): string {
|
||||
return `src_${Date.now().toString(36)}_${idCounter}`;
|
||||
}
|
||||
|
||||
/** A demo token for a newly created webhook's server-generated id/secret (mock only). */
|
||||
function randomToken(): string {
|
||||
return (
|
||||
Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2)
|
||||
);
|
||||
}
|
||||
|
||||
function refsFor(id: string): SourcePolicyRef[] {
|
||||
return references[id] ?? [];
|
||||
}
|
||||
|
||||
/** Mirror the backend's secret redaction so config rows never surface a secret in the overview. */
|
||||
function configRows(options: Record<string, unknown>) {
|
||||
const secret = /secret|password|token/i;
|
||||
return Object.entries(options).map(([key, value]) => ({
|
||||
label: key.charAt(0).toUpperCase() + key.slice(1),
|
||||
value: String(value),
|
||||
value: secret.test(key) ? "********" : String(value),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -120,6 +142,7 @@ function toSourceView(
|
||||
refs: SourcePolicyRef[],
|
||||
): SourceView {
|
||||
const docs = docsFor(source.id);
|
||||
const webhookId = source.options.webhookId;
|
||||
return {
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
@@ -131,6 +154,10 @@ function toSourceView(
|
||||
docsTotal: docs.total,
|
||||
docs24h: docs.last24h,
|
||||
docs30d: docs.last30d,
|
||||
webhookPath:
|
||||
source.type === "webhook" && typeof webhookId === "string"
|
||||
? `/api/v1/webhooks/${webhookId}`
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -207,8 +234,18 @@ export const sourcesHandlers = [
|
||||
? store.find((s) => s.id === incoming.id)
|
||||
: undefined;
|
||||
const id = existing?.id ?? nextId();
|
||||
// A new webhook's id + secret are minted server-side and revealed once on this create response.
|
||||
let options = incoming.options ?? {};
|
||||
if (incoming.type === "webhook" && !existing && !options.webhookId) {
|
||||
options = {
|
||||
...options,
|
||||
webhookId: `whk_${randomToken()}`,
|
||||
signingSecret: `whsec_${randomToken()}`,
|
||||
};
|
||||
}
|
||||
const saved: StoredSource = {
|
||||
...incoming,
|
||||
options,
|
||||
id,
|
||||
owner: existing?.owner ?? "you@acme.com",
|
||||
};
|
||||
|
||||
@@ -85,3 +85,27 @@
|
||||
justify-content: flex-end;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
/* Webhook delivery URL / signing secret: read-only value + copy button. */
|
||||
.portal-source-builder__copy-row {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.portal-source-builder__copy-row > :first-child {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.portal-source-builder__reveal {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.portal-source-builder__muted {
|
||||
margin: 0;
|
||||
font-size: 0.8125rem;
|
||||
color: var(--color-text-4);
|
||||
}
|
||||
|
||||
@@ -102,6 +102,39 @@ describe("SourceBuilder", () => {
|
||||
).toBeDisabled();
|
||||
});
|
||||
|
||||
it("reveals the delivery URL and signing secret once after creating a webhook", async () => {
|
||||
createSource.mockResolvedValue({
|
||||
id: "wh-1",
|
||||
options: { webhookId: "whk_abc123", signingSecret: "whsec_topsecret" },
|
||||
});
|
||||
renderBuilder("/processor/sources/new");
|
||||
|
||||
fireEvent.change(screen.getByLabelText(/portal\.sources\.wizard\.name/), {
|
||||
target: { value: "Partner uploads" },
|
||||
});
|
||||
// Webhook's connection is optional (self-hosted local-disk), so a name is enough to create.
|
||||
fireEvent.click(screen.getByText("portal.sources.types.webhook.label"));
|
||||
fireEvent.click(screen.getByText("portal.sources.builder.create"));
|
||||
|
||||
await waitFor(() => expect(createSource).toHaveBeenCalledTimes(1));
|
||||
expect(createSource).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ type: "webhook", name: "Partner uploads" }),
|
||||
);
|
||||
|
||||
expect(
|
||||
await screen.findByDisplayValue("whsec_topsecret"),
|
||||
).toBeInTheDocument();
|
||||
expect(
|
||||
screen.getByDisplayValue(/\/api\/v1\/webhooks\/whk_abc123$/),
|
||||
).toBeInTheDocument();
|
||||
expect(screen.queryByText("sources list")).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(
|
||||
screen.getByText("portal.sources.types.webhook.reveal.done"),
|
||||
);
|
||||
expect(await screen.findByText("sources list")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("blocks create until required fields are filled", async () => {
|
||||
renderBuilder("/processor/sources/new");
|
||||
// Name given but directory (required) still blank -> Create disabled.
|
||||
|
||||
@@ -27,11 +27,17 @@ import {
|
||||
CREATABLE_SOURCE_TYPES,
|
||||
defaultOptions,
|
||||
sourceTypeMeta,
|
||||
WEBHOOK_SOURCE_TYPE,
|
||||
type CreatableSourceType,
|
||||
} from "@portal/components/sources/sourceTypes";
|
||||
import { S3ConnectionPicker } from "@portal/components/sources/S3ConnectionPicker";
|
||||
import "@portal/views/SourceBuilder.css";
|
||||
|
||||
/** Absolute delivery URL a sender POSTs to for a webhook's routing id. */
|
||||
function webhookUrl(webhookId: string): string {
|
||||
return `${window.location.origin}/api/v1/webhooks/${webhookId}`;
|
||||
}
|
||||
|
||||
const OFFERED_TYPES = creatableSourceTypes();
|
||||
|
||||
/** A source's stored type resolved to its create-form metadata (edit falls back to any type). */
|
||||
@@ -84,6 +90,11 @@ export function SourceBuilder() {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [pendingDelete, setPendingDelete] = useState(false);
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
// Set after a webhook is created: its one-time delivery id + signing secret, shown before leaving.
|
||||
const [reveal, setReveal] = useState<{
|
||||
webhookId: string;
|
||||
secret: string;
|
||||
} | null>(null);
|
||||
|
||||
// Seed once: immediately for a new source, or after the record loads for edit.
|
||||
useEffect(() => {
|
||||
@@ -112,18 +123,40 @@ export function SourceBuilder() {
|
||||
);
|
||||
const canSave = name.trim() !== "" && requiredComplete && !submitting;
|
||||
|
||||
// An existing webhook's delivery URL, read-only in edit (the secret is revealed only at creation).
|
||||
const editingWebhookId =
|
||||
isEdit && sourceState.data?.type === WEBHOOK_SOURCE_TYPE
|
||||
? String(sourceState.data.options?.webhookId ?? "")
|
||||
: "";
|
||||
const revealUrl = reveal ? webhookUrl(reveal.webhookId) : "";
|
||||
const revealSecret = reveal ? reveal.secret : "";
|
||||
|
||||
function dismissReveal() {
|
||||
setReveal(null);
|
||||
navigate(listPath);
|
||||
}
|
||||
|
||||
async function save() {
|
||||
if (!canSave) return;
|
||||
setSubmitting(true);
|
||||
setError(null);
|
||||
try {
|
||||
await createSource({
|
||||
const saved = await createSource({
|
||||
id: isEdit ? id : undefined,
|
||||
name: name.trim(),
|
||||
type: type.type,
|
||||
options,
|
||||
enabled,
|
||||
});
|
||||
// A new webhook returns its minted id + secret once; reveal them (with the URL) before leaving.
|
||||
if (!isEdit && type.type === WEBHOOK_SOURCE_TYPE) {
|
||||
const webhookId = String(saved.options?.webhookId ?? "");
|
||||
const secret = String(saved.options?.signingSecret ?? "");
|
||||
if (webhookId && secret) {
|
||||
setReveal({ webhookId, secret });
|
||||
return;
|
||||
}
|
||||
}
|
||||
navigate(listPath);
|
||||
} catch (e) {
|
||||
setError(errorMessage(e));
|
||||
@@ -131,6 +164,10 @@ export function SourceBuilder() {
|
||||
}
|
||||
}
|
||||
|
||||
function copy(text: string) {
|
||||
void navigator.clipboard?.writeText(text);
|
||||
}
|
||||
|
||||
async function confirmDelete() {
|
||||
if (!id || deleting) return;
|
||||
setDeleting(true);
|
||||
@@ -290,6 +327,28 @@ export function SourceBuilder() {
|
||||
</FormField>
|
||||
))}
|
||||
|
||||
{editingWebhookId && (
|
||||
<FormField
|
||||
label={t("portal.sources.types.webhook.detail.deliveryUrl")}
|
||||
helperText={t("portal.sources.types.webhook.detail.secretNote")}
|
||||
>
|
||||
<div className="portal-source-builder__copy-row">
|
||||
<Input
|
||||
value={webhookUrl(editingWebhookId)}
|
||||
readOnly
|
||||
onFocus={(e) => e.currentTarget.select()}
|
||||
/>
|
||||
<Button
|
||||
variant="tertiary"
|
||||
size="sm"
|
||||
onClick={() => copy(webhookUrl(editingWebhookId))}
|
||||
>
|
||||
{t("portal.sources.types.webhook.reveal.copy")}
|
||||
</Button>
|
||||
</div>
|
||||
</FormField>
|
||||
)}
|
||||
|
||||
{error && <Banner tone="danger" description={error} />}
|
||||
</div>
|
||||
|
||||
@@ -321,6 +380,69 @@ export function SourceBuilder() {
|
||||
>
|
||||
<p>{t("portal.sources.delete.body", { name })}</p>
|
||||
</Modal>
|
||||
|
||||
<Modal
|
||||
open={reveal !== null}
|
||||
onClose={dismissReveal}
|
||||
width="md"
|
||||
title={t("portal.sources.types.webhook.reveal.title")}
|
||||
footer={
|
||||
<div className="portal-source-builder__delete-actions">
|
||||
<Button size="sm" onClick={dismissReveal}>
|
||||
{t("portal.sources.types.webhook.reveal.done")}
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
{reveal && (
|
||||
<div className="portal-source-builder__reveal">
|
||||
<Banner
|
||||
tone="warning"
|
||||
description={t(
|
||||
"portal.sources.types.webhook.reveal.secretWarning",
|
||||
)}
|
||||
/>
|
||||
<FormField label={t("portal.sources.types.webhook.reveal.url")}>
|
||||
<div className="portal-source-builder__copy-row">
|
||||
<Input
|
||||
value={revealUrl}
|
||||
readOnly
|
||||
onFocus={(e) => e.currentTarget.select()}
|
||||
/>
|
||||
<Button
|
||||
variant="tertiary"
|
||||
size="sm"
|
||||
onClick={() => copy(revealUrl)}
|
||||
>
|
||||
{t("portal.sources.types.webhook.reveal.copy")}
|
||||
</Button>
|
||||
</div>
|
||||
</FormField>
|
||||
<FormField
|
||||
label={t("portal.sources.types.webhook.reveal.secret")}
|
||||
helperText={t("portal.sources.types.webhook.reveal.secretHelp")}
|
||||
>
|
||||
<div className="portal-source-builder__copy-row">
|
||||
<Input
|
||||
value={revealSecret}
|
||||
readOnly
|
||||
onFocus={(e) => e.currentTarget.select()}
|
||||
/>
|
||||
<Button
|
||||
variant="tertiary"
|
||||
size="sm"
|
||||
onClick={() => copy(revealSecret)}
|
||||
>
|
||||
{t("portal.sources.types.webhook.reveal.copy")}
|
||||
</Button>
|
||||
</div>
|
||||
</FormField>
|
||||
<p className="portal-source-builder__muted">
|
||||
{t("portal.sources.types.webhook.reveal.usage")}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user