Add cluster deployment artifacts and Helm chart

app/common/build.gradle

@@ -78,6 +78,9 @@ dependencies {
         runtimeOnly "com.stirling:jpdfium-natives-${platform}:1.0.1"
     }
 
+    // Bucket4j (local in-process token bucket for RateLimitStore default impl)
+    implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
+
     // ArchUnit: enforces module dependency direction (see ArchitectureTest)
     testImplementation 'com.tngtech.archunit:archunit-junit5:1.4.2'
 }

app/common/src/main/java/stirling/software/common/cluster/ClusterBackplane.java

@@ -0,0 +1,24 @@
+package stirling.software.common.cluster;
+
+/** Health and identity facade for the active cluster backplane. */
+public interface ClusterBackplane {
+
+    /** Returns {@code true} when the backplane is reachable; used for health endpoints. */
+    boolean isHealthy();
+
+    /** Returns {@code "inprocess"} or {@code "valkey"}. */
+    String backplaneType();
+
+    /** Returns this JVM's stable node id (matches {@code Cluster.resolvedNodeId()}). */
+    String localNodeId();
+
+    /**
+     * Whether this JVM should run the local {@link
+     * stirling.software.common.service.TaskManager#cleanupOldJobs()} loop. Distributed backplanes
+     * own job expiry via their own TTL, so they should override this to return {@code false}.
+     * Defaults to {@code true} so in-process behavior is preserved without an explicit override.
+     */
+    default boolean shouldRunLocalCleanup() {
+        return true;
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/ClusterConfig.java

@@ -0,0 +1,63 @@
+package stirling.software.common.cluster;
+
+import org.springframework.context.annotation.Configuration;
+
+import jakarta.annotation.PostConstruct;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.ApplicationProperties.Cluster;
+
+/**
+ * Validates that cluster mode is internally consistent.
+ *
+ * <p>Cluster settings are bound on the central {@link ApplicationProperties} under {@code
+ * cluster.*}; this class reads {@link ApplicationProperties#getCluster()} and runs guards in {@link
+ * PostConstruct}. When {@code cluster.enabled=false} (the default) all checks are skipped so a
+ * single-instance install needs no new config.
+ */
+@Slf4j
+@Configuration
+@RequiredArgsConstructor
+public class ClusterConfig {
+
+    private final ApplicationProperties applicationProperties;
+
+    @PostConstruct
+    void validate() {
+        Cluster cluster = applicationProperties.getCluster();
+        if (!cluster.isEnabled()) {
+            return;
+        }
+        String backplane = cluster.getBackplane();
+        if ("valkey".equalsIgnoreCase(backplane)) {
+            String url = cluster.getValkey() == null ? null : cluster.getValkey().getUrl();
+            if (url == null || url.isBlank()) {
+                throw new IllegalStateException(
+                        "cluster.enabled=true with backplane=valkey requires"
+                                + " cluster.valkey.url to be set (e.g."
+                                + " redis://valkey:6379).");
+            }
+        } else if ("inprocess".equalsIgnoreCase(backplane)) {
+            // enabled+inprocess only coordinates the local JVM; cross-node lookups will 410.
+            log.warn(
+                    "cluster.enabled=true with backplane=inprocess - only the local"
+                            + " JVM is coordinated. Cross-node lookups and the file proxy will fail."
+                            + " Use backplane=valkey for real multi-node deployments.");
+        } else {
+            // Fail fast on typos like "valky" so Spring doesn't later report a cryptic
+            // "no ClusterBackplane bean" - the operator-facing error names the bad value.
+            throw new IllegalStateException(
+                    "cluster.enabled=true with unknown backplane '"
+                            + backplane
+                            + "'. Valid values: inprocess | valkey.");
+        }
+        log.info(
+                "Cluster mode enabled (backplane={}, nodeRole={}, nodeId={}).",
+                backplane,
+                cluster.resolvedRole(),
+                cluster.resolvedNodeId());
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/ClusterNode.java

@@ -0,0 +1,12 @@
+package stirling.software.common.cluster;
+
+import java.time.Instant;
+
+/**
+ * Snapshot of a peer node as recorded in the {@link InstanceRegistry}.
+ *
+ * @param internalAddress {@code host:port} the node listens on for {@code /internal/cluster/**}
+ * @param role one of {@code WEB}, {@code WORKER}, {@code BOTH}
+ */
+public record ClusterNode(
+        String nodeId, String internalAddress, Instant lastHeartbeat, String role) {}

app/common/src/main/java/stirling/software/common/cluster/DistributedLock.java

@@ -0,0 +1,21 @@
+package stirling.software.common.cluster;
+
+import java.time.Duration;
+import java.util.Optional;
+
+/** Cluster-wide mutual exclusion primitive; non-reentrant by contract. */
+public interface DistributedLock {
+
+    Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime);
+
+    interface LockHandle extends AutoCloseable {
+        void release();
+
+        boolean renew(Duration leaseTime);
+
+        @Override
+        default void close() {
+            release();
+        }
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/FileStore.java

@@ -0,0 +1,45 @@
+package stirling.software.common.cluster;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Low-level storage seam for result/job files. */
+public interface FileStore {
+
+    /** Stored file record. */
+    record Stored(String fileId, long size) {}
+
+    /** Store the given stream and return a generated file id and total bytes written. */
+    Stored store(InputStream in, String originalName) throws IOException;
+
+    /**
+     * Store the file at {@code source} and return a generated file id and total bytes written.
+     *
+     * <p>Default implementation opens {@code source} as a stream and delegates to {@link
+     * #store(InputStream, String)}. Local-disk implementations should override to use a direct
+     * file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on Linux),
+     * which avoids the two-memory-copy hit of streaming a disk-backed upload through the JVM heap.
+     */
+    default Stored store(Path source, String originalName) throws IOException {
+        try (InputStream in = Files.newInputStream(source)) {
+            return store(in, originalName);
+        }
+    }
+
+    /** Open the stored file for streaming reads. Caller closes. */
+    InputStream retrieve(String fileId) throws IOException;
+
+    /** Load the stored file into a byte array. */
+    byte[] retrieveBytes(String fileId) throws IOException;
+
+    /** Size of the stored file in bytes. */
+    long size(String fileId) throws IOException;
+
+    /** Delete the stored file. Returns true if a file was removed. */
+    boolean delete(String fileId);
+
+    /** Whether the file id exists in the store. */
+    boolean exists(String fileId);
+}

app/common/src/main/java/stirling/software/common/cluster/InstanceRegistry.java

@@ -0,0 +1,18 @@
+package stirling.software.common.cluster;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Optional;
+
+/** Maps {@code nodeId} to its internal cluster address, with TTL'd heartbeats. */
+public interface InstanceRegistry {
+
+    /** Register or refresh this node. Idempotent so a wiped backplane self-heals on next tick. */
+    void register(ClusterNode node, Duration heartbeatTtl);
+
+    Optional<ClusterNode> lookup(String nodeId);
+
+    Collection<ClusterNode> activeNodes();
+
+    void deregister(String nodeId);
+}

app/common/src/main/java/stirling/software/common/cluster/JobStore.java

@@ -0,0 +1,24 @@
+package stirling.software.common.cluster;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Optional;
+
+/** Cluster-visible storage for job status and result metadata, with TTL'd entries. */
+public interface JobStore {
+
+    /** Persist or overwrite a job entry. {@code ttl} sets the lifetime of the entry. */
+    void put(JobStoreEntry entry, Duration ttl);
+
+    Optional<JobStoreEntry> get(String jobId);
+
+    void delete(String jobId);
+
+    boolean exists(String jobId);
+
+    /** Reverse lookup: which job owns this result file id? */
+    Optional<String> findJobIdByFileId(String fileId);
+
+    /** Snapshot of every active entry. Used by admin/stats endpoints; may be O(n). */
+    Collection<JobStoreEntry> all();
+}

app/common/src/main/java/stirling/software/common/cluster/JobStoreEntry.java

@@ -0,0 +1,30 @@
+package stirling.software.common.cluster;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Cluster-visible projection of a job's status and result metadata, as persisted in {@link
+ * JobStore}.
+ *
+ * @param owningNodeId the node id that originally executed the job
+ */
+public record JobStoreEntry(
+        String jobId,
+        JobState state,
+        String owningNodeId,
+        Instant createdAt,
+        Instant completedAt,
+        String error,
+        List<String> fileIds,
+        Map<String, String> resultMeta) {
+
+    /** Lifecycle states for a job as observed by the cluster. */
+    public enum JobState {
+        PENDING,
+        RUNNING,
+        COMPLETE,
+        FAILED
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/KeyValueCache.java

@@ -0,0 +1,16 @@
+package stirling.software.common.cluster;
+
+import java.time.Duration;
+import java.util.Optional;
+
+/** Short-TTL namespaced key/value cache backed by the cluster backplane. */
+public interface KeyValueCache {
+
+    void put(String namespace, String key, String value, Duration ttl);
+
+    Optional<String> get(String namespace, String key);
+
+    void evict(String namespace, String key);
+
+    void evictNamespace(String namespace);
+}

app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java

@@ -0,0 +1,18 @@
+package stirling.software.common.cluster;
+
+import java.time.Duration;
+
+/** Token-bucket rate limiting backed by the cluster backplane. */
+public interface RateLimitStore {
+
+    /**
+     * Attempt to consume one token from the bucket identified by {@code bucketKey}.
+     *
+     * @param bucketKey opaque key identifying the bucket (e.g. {@code api:user:123})
+     * @param capacity bucket capacity
+     * @param refillPeriod time window over which {@code capacity} tokens refill
+     */
+    RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod);
+
+    record RateLimitDecision(boolean allowed, long remainingTokens, long nanosToWaitForRefill) {}
+}

app/common/src/main/java/stirling/software/common/cluster/StickyMissRecorder.java

@@ -0,0 +1,7 @@
+package stirling.software.common.cluster;
+
+/** Records one increment per sticky-session miss (a 410 Gone for a job owned by another node). */
+@FunctionalInterface
+public interface StickyMissRecorder {
+    void recordStickyMiss();
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessClusterBackplane.java

@@ -0,0 +1,31 @@
+package stirling.software.common.cluster.inprocess;
+
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.model.ApplicationProperties;
+
+@Slf4j
+public class InProcessClusterBackplane implements ClusterBackplane {
+
+    private final ApplicationProperties applicationProperties;
+
+    public InProcessClusterBackplane(ApplicationProperties applicationProperties) {
+        this.applicationProperties = applicationProperties;
+    }
+
+    @Override
+    public boolean isHealthy() {
+        return true;
+    }
+
+    @Override
+    public String backplaneType() {
+        return "inprocess";
+    }
+
+    @Override
+    public String localNodeId() {
+        return applicationProperties.getCluster().resolvedNodeId();
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessClusterConfiguration.java

@@ -0,0 +1,65 @@
+package stirling.software.common.cluster.inprocess;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.DistributedLock;
+import stirling.software.common.cluster.InstanceRegistry;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.KeyValueCache;
+import stirling.software.common.cluster.RateLimitStore;
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Default cluster backplane wiring: every interface gets an {@code InProcess*} bean. Active when
+ * cluster mode is off or {@code cluster.backplane=inprocess}.
+ */
+@Slf4j
+@Configuration
+@ConditionalOnExpression(
+        "!${cluster.enabled:false} ||"
+                + " '${cluster.backplane:inprocess}'.equalsIgnoreCase('inprocess')")
+public class InProcessClusterConfiguration {
+
+    @Bean
+    @ConditionalOnMissingBean
+    public ClusterBackplane clusterBackplane(ApplicationProperties applicationProperties) {
+        log.info("Cluster backplane: in-process (single node)");
+        return new InProcessClusterBackplane(applicationProperties);
+    }
+
+    @Bean
+    @ConditionalOnMissingBean
+    public JobStore jobStore() {
+        return new InProcessJobStore();
+    }
+
+    @Bean
+    @ConditionalOnMissingBean
+    public RateLimitStore rateLimitStore() {
+        return new InProcessRateLimitStore();
+    }
+
+    @Bean
+    @ConditionalOnMissingBean
+    public DistributedLock distributedLock() {
+        return new InProcessDistributedLock();
+    }
+
+    @Bean
+    @ConditionalOnMissingBean
+    public KeyValueCache keyValueCache() {
+        return new InProcessKeyValueCache();
+    }
+
+    @Bean
+    @ConditionalOnMissingBean
+    public InstanceRegistry instanceRegistry() {
+        return new InProcessInstanceRegistry();
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessDistributedLock.java

@@ -0,0 +1,128 @@
+package stirling.software.common.cluster.inprocess;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import stirling.software.common.cluster.DistributedLock;
+
+/**
+ * In-process {@link DistributedLock}, non-reentrant per the interface contract, with lease-expiry
+ * semantics that mirror a SET-NX-EX style distributed backend.
+ *
+ * <p>Each lock state carries a per-acquire {@code ownerToken} and an {@code expiryNanos}; another
+ * caller can take over once the lease has elapsed even if the original holder never called {@link
+ * LockHandle#release()}. This matters mostly for parity with the Valkey-backed implementation
+ * (Redis {@code SETEX} auto-expires the key); within a single JVM a crashed holder takes its lock
+ * state with it, but tests and code that rely on the {@code leaseTime} parameter still need it to
+ * be honored.
+ *
+ * <p>Expiry is lazy: an expired lock state lingers in the map until the next acquire attempt for
+ * the same key replaces it. Per-key cleanup also happens on explicit {@link LockHandle#release()},
+ * so a balanced acquire/release workload keeps the map size bounded.
+ */
+public class InProcessDistributedLock implements DistributedLock {
+
+    private final ConcurrentHashMap<String, LockState> locks = new ConcurrentHashMap<>();
+    private final AtomicLong tokenSeq = new AtomicLong();
+
+    /**
+     * Lease state for a single acquired lock. {@code ownerToken} prevents a former holder from
+     * releasing or renewing a lock now owned by someone else after lease expiry; {@code
+     * expiryNanos} is read/written only inside {@link ConcurrentHashMap#compute} so the bin lock
+     * provides the necessary happens-before guarantee.
+     */
+    private static final class LockState {
+        final long ownerToken;
+        long expiryNanos;
+
+        LockState(long ownerToken, long expiryNanos) {
+            this.ownerToken = ownerToken;
+            this.expiryNanos = expiryNanos;
+        }
+    }
+
+    @Override
+    public Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime) {
+        long token = tokenSeq.incrementAndGet();
+        long nowNanos = System.nanoTime();
+        long expiryNanos = nowNanos + leaseTime.toNanos();
+        boolean[] acquired = {false};
+        locks.compute(
+                lockKey,
+                (k, existing) -> {
+                    if (existing == null || existing.expiryNanos - nowNanos <= 0L) {
+                        // No lock, or the previous lease has expired - we take it. Subtraction
+                        // form avoids the long-overflow trap that would bite a naive
+                        // expiryNanos <= nowNanos comparison around System.nanoTime() rollover.
+                        acquired[0] = true;
+                        return new LockState(token, expiryNanos);
+                    }
+                    return existing;
+                });
+        if (!acquired[0]) {
+            return Optional.empty();
+        }
+        return Optional.of(new InProcessHandle(lockKey, token));
+    }
+
+    private void releaseInternal(String lockKey, long token) {
+        locks.compute(
+                lockKey,
+                (k, existing) -> {
+                    if (existing == null || existing.ownerToken != token) {
+                        // Already removed, expired-and-replaced, or never ours.
+                        return existing;
+                    }
+                    return null;
+                });
+    }
+
+    private boolean renewInternal(String lockKey, long token, Duration leaseTime) {
+        long nowNanos = System.nanoTime();
+        boolean[] renewed = {false};
+        locks.compute(
+                lockKey,
+                (k, existing) -> {
+                    if (existing == null
+                            || existing.ownerToken != token
+                            || existing.expiryNanos - nowNanos <= 0L) {
+                        // Lock is gone or expired; renewal is a no-op so the caller can detect it.
+                        return existing;
+                    }
+                    existing.expiryNanos = nowNanos + leaseTime.toNanos();
+                    renewed[0] = true;
+                    return existing;
+                });
+        return renewed[0];
+    }
+
+    private final class InProcessHandle implements LockHandle {
+        private final String lockKey;
+        private final long token;
+        private boolean released;
+
+        InProcessHandle(String lockKey, long token) {
+            this.lockKey = lockKey;
+            this.token = token;
+        }
+
+        @Override
+        public synchronized void release() {
+            if (released) {
+                return;
+            }
+            released = true;
+            releaseInternal(lockKey, token);
+        }
+
+        @Override
+        public synchronized boolean renew(Duration leaseTime) {
+            if (released) {
+                return false;
+            }
+            return renewInternal(lockKey, token, leaseTime);
+        }
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessInstanceRegistry.java

@@ -0,0 +1,42 @@
+package stirling.software.common.cluster.inprocess;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.InstanceRegistry;
+
+public class InProcessInstanceRegistry implements InstanceRegistry {
+
+    private final AtomicReference<ClusterNode> self = new AtomicReference<>();
+
+    @Override
+    public void register(ClusterNode node, Duration heartbeatTtl) {
+        self.set(node);
+    }
+
+    @Override
+    public Optional<ClusterNode> lookup(String nodeId) {
+        ClusterNode current = self.get();
+        return current != null && current.nodeId().equals(nodeId)
+                ? Optional.of(current)
+                : Optional.empty();
+    }
+
+    @Override
+    public Collection<ClusterNode> activeNodes() {
+        ClusterNode current = self.get();
+        return current == null ? Collections.emptyList() : Collections.singletonList(current);
+    }
+
+    @Override
+    public void deregister(String nodeId) {
+        ClusterNode current = self.get();
+        if (current != null && current.nodeId().equals(nodeId)) {
+            self.set(null);
+        }
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessJobStore.java

@@ -0,0 +1,96 @@
+package stirling.software.common.cluster.inprocess;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+
+@Slf4j
+public class InProcessJobStore implements JobStore {
+
+    private final ConcurrentHashMap<String, Holder> entries = new ConcurrentHashMap<>();
+
+    @Override
+    public void put(JobStoreEntry entry, Duration ttl) {
+        Instant expiry = ttl == null ? Instant.MAX : Instant.now().plus(ttl);
+        entries.put(entry.jobId(), new Holder(entry, expiry));
+    }
+
+    @Override
+    public Optional<JobStoreEntry> get(String jobId) {
+        Holder h = entries.get(jobId);
+        if (h == null) {
+            return Optional.empty();
+        }
+        if (h.isExpired()) {
+            entries.remove(jobId, h);
+            return Optional.empty();
+        }
+        return Optional.of(h.entry);
+    }
+
+    @Override
+    public void delete(String jobId) {
+        entries.remove(jobId);
+    }
+
+    @Override
+    public boolean exists(String jobId) {
+        return get(jobId).isPresent();
+    }
+
+    @Override
+    public Optional<String> findJobIdByFileId(String fileId) {
+        for (Map.Entry<String, Holder> e : entries.entrySet()) {
+            Holder h = e.getValue();
+            if (h.isExpired()) {
+                continue;
+            }
+            List<String> fileIds = h.entry.fileIds();
+            if (fileIds != null && fileIds.contains(fileId)) {
+                return Optional.of(e.getKey());
+            }
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public Collection<JobStoreEntry> all() {
+        List<JobStoreEntry> result = new ArrayList<>(entries.size());
+        for (Holder h : entries.values()) {
+            if (!h.isExpired()) {
+                result.add(h.entry);
+            }
+        }
+        return result;
+    }
+
+    /** Drop entries whose TTL has elapsed. Called by the {@code TaskManager} cleanup scheduler. */
+    public int purgeExpired() {
+        int removed = 0;
+        Instant now = Instant.now();
+        for (Map.Entry<String, Holder> e : entries.entrySet()) {
+            if (!e.getValue().expiry.equals(Instant.MAX) && e.getValue().expiry.isBefore(now)) {
+                if (entries.remove(e.getKey(), e.getValue())) {
+                    removed++;
+                }
+            }
+        }
+        return removed;
+    }
+
+    private record Holder(JobStoreEntry entry, Instant expiry) {
+        boolean isExpired() {
+            return !expiry.equals(Instant.MAX) && expiry.isBefore(Instant.now());
+        }
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessKeyValueCache.java

@@ -0,0 +1,55 @@
+package stirling.software.common.cluster.inprocess;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+import stirling.software.common.cluster.KeyValueCache;
+
+public class InProcessKeyValueCache implements KeyValueCache {
+
+    private final ConcurrentHashMap<String, ConcurrentHashMap<String, Expiring>> namespaces =
+            new ConcurrentHashMap<>();
+
+    @Override
+    public void put(String namespace, String key, String value, Duration ttl) {
+        Instant expiry = ttl == null ? Instant.MAX : Instant.now().plus(ttl);
+        namespaces
+                .computeIfAbsent(namespace, n -> new ConcurrentHashMap<>())
+                .put(key, new Expiring(value, expiry));
+    }
+
+    @Override
+    public Optional<String> get(String namespace, String key) {
+        Map<String, Expiring> ns = namespaces.get(namespace);
+        if (ns == null) {
+            return Optional.empty();
+        }
+        Expiring e = ns.get(key);
+        if (e == null) {
+            return Optional.empty();
+        }
+        if (e.expiry.isBefore(Instant.now())) {
+            ns.remove(key, e);
+            return Optional.empty();
+        }
+        return Optional.of(e.value);
+    }
+
+    @Override
+    public void evict(String namespace, String key) {
+        Map<String, Expiring> ns = namespaces.get(namespace);
+        if (ns != null) {
+            ns.remove(key);
+        }
+    }
+
+    @Override
+    public void evictNamespace(String namespace) {
+        namespaces.remove(namespace);
+    }
+
+    private record Expiring(String value, Instant expiry) {}
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/InProcessRateLimitStore.java

@@ -0,0 +1,49 @@
+package stirling.software.common.cluster.inprocess;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import io.github.bucket4j.Bandwidth;
+import io.github.bucket4j.Bucket;
+import io.github.bucket4j.ConsumptionProbe;
+import io.github.bucket4j.local.LocalBucketBuilder;
+
+import stirling.software.common.cluster.RateLimitStore;
+
+/** Bucket4j-backed token bucket implementation of {@link RateLimitStore}. */
+public class InProcessRateLimitStore implements RateLimitStore {
+
+    /** Cap to bound memory; oldest accessed buckets are evicted. */
+    private static final int MAX_BUCKETS = 10_000;
+
+    private final Map<String, Bucket> buckets =
+            Collections.synchronizedMap(
+                    new LinkedHashMap<String, Bucket>(256, 0.75f, true) {
+                        @Override
+                        protected boolean removeEldestEntry(Map.Entry<String, Bucket> eldest) {
+                            return size() > MAX_BUCKETS;
+                        }
+                    });
+
+    @Override
+    public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) {
+        String compositeKey = bucketKey + "|" + capacity + "|" + refillPeriod.toNanos();
+        Bucket bucket =
+                buckets.computeIfAbsent(compositeKey, k -> buildBucket(capacity, refillPeriod));
+        ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1);
+        return new RateLimitDecision(
+                probe.isConsumed(),
+                probe.getRemainingTokens(),
+                probe.isConsumed() ? 0L : probe.getNanosToWaitForRefill());
+    }
+
+    private static Bucket buildBucket(long capacity, Duration refillPeriod) {
+        Bandwidth limit =
+                Bandwidth.builder().capacity(capacity).refillGreedy(capacity, refillPeriod).build();
+        LocalBucketBuilder builder = Bucket.builder();
+        builder.addLimit(limit);
+        return builder.build();
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/LocalDiskFileStore.java

@@ -0,0 +1,128 @@
+package stirling.software.common.cluster.inprocess;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.FileStore;
+
+/** Local-disk {@link FileStore} storing files under a base directory keyed by a UUID file id. */
+@Slf4j
+public class LocalDiskFileStore implements FileStore {
+
+    private final String baseDirPath;
+
+    public LocalDiskFileStore(String baseDirPath) {
+        this.baseDirPath = baseDirPath;
+    }
+
+    @Override
+    public Stored store(InputStream in, String originalName) throws IOException {
+        String fileId = UUID.randomUUID().toString();
+        Path filePath = resolve(fileId);
+        Files.createDirectories(filePath.getParent());
+        boolean success = false;
+        try {
+            long size = Files.copy(in, filePath);
+            success = true;
+            return new Stored(fileId, size);
+        } finally {
+            if (!success) {
+                try {
+                    Files.deleteIfExists(filePath);
+                } catch (IOException cleanupEx) {
+                    log.warn(
+                            "Failed to clean up partial file {} after store failure",
+                            filePath,
+                            cleanupEx);
+                }
+            }
+        }
+    }
+
+    /**
+     * File-to-file copy. {@link Files#copy(Path, Path, java.nio.file.CopyOption...)} can use {@code
+     * sendfile(2)} on Linux for a zero-copy kernel transfer when source and destination share a
+     * filesystem, avoiding the streaming overhead of pulling the bytes through the JVM heap. Reads
+     * the source size before copying so the post-copy stat is unnecessary.
+     */
+    @Override
+    public Stored store(Path source, String originalName) throws IOException {
+        String fileId = UUID.randomUUID().toString();
+        Path filePath = resolve(fileId);
+        Files.createDirectories(filePath.getParent());
+        long size = Files.size(source);
+        boolean success = false;
+        try {
+            Files.copy(source, filePath);
+            success = true;
+            return new Stored(fileId, size);
+        } finally {
+            if (!success) {
+                try {
+                    Files.deleteIfExists(filePath);
+                } catch (IOException cleanupEx) {
+                    log.warn(
+                            "Failed to clean up partial file {} after store failure",
+                            filePath,
+                            cleanupEx);
+                }
+            }
+        }
+    }
+
+    @Override
+    public InputStream retrieve(String fileId) throws IOException {
+        return new BufferedInputStream(Files.newInputStream(resolve(fileId)));
+    }
+
+    @Override
+    public byte[] retrieveBytes(String fileId) throws IOException {
+        Path filePath = resolve(fileId);
+        if (!Files.exists(filePath)) {
+            throw new IOException("File not found with ID: " + fileId);
+        }
+        return Files.readAllBytes(filePath);
+    }
+
+    @Override
+    public long size(String fileId) throws IOException {
+        Path filePath = resolve(fileId);
+        if (!Files.exists(filePath)) {
+            throw new IOException("File not found with ID: " + fileId);
+        }
+        return Files.size(filePath);
+    }
+
+    @Override
+    public boolean delete(String fileId) {
+        try {
+            return Files.deleteIfExists(resolve(fileId));
+        } catch (IOException e) {
+            log.error("Error deleting file with ID: {}", fileId, e);
+            return false;
+        }
+    }
+
+    @Override
+    public boolean exists(String fileId) {
+        return Files.exists(resolve(fileId));
+    }
+
+    public Path resolve(String fileId) {
+        if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
+            throw new IllegalArgumentException("Invalid file ID");
+        }
+        Path basePath = Path.of(baseDirPath).normalize().toAbsolutePath();
+        Path resolvedPath = basePath.resolve(fileId).normalize();
+        if (!resolvedPath.startsWith(basePath)) {
+            throw new IllegalArgumentException("File ID resolves to an invalid path");
+        }
+        return resolvedPath;
+    }
+}

app/common/src/main/java/stirling/software/common/cluster/inprocess/LocalDiskFileStoreConfiguration.java

@@ -0,0 +1,29 @@
+package stirling.software.common.cluster.inprocess;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import stirling.software.common.cluster.FileStore;
+
+/**
+ * Always-on wiring for the per-node local-disk {@link FileStore}. Active when {@code
+ * cluster.artifactStore=local} (the default; {@code matchIfMissing=true}). The S3 artifact-store
+ * supplies its own bean when {@code cluster.artifactStore=s3}.
+ */
+@Configuration
+@ConditionalOnProperty(
+        prefix = "cluster",
+        name = "artifactStore",
+        havingValue = "local",
+        matchIfMissing = true)
+public class LocalDiskFileStoreConfiguration {
+
+    @Bean
+    @ConditionalOnMissingBean
+    public FileStore fileStore(@Value("${stirling.tempDir:/tmp/stirling-files}") String tempDir) {
+        return new LocalDiskFileStore(tempDir);
+    }
+}

app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java

@@ -13,6 +13,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
+import java.util.UUID;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
@@ -77,6 +78,7 @@ public class ApplicationProperties {
     private PdfEditor pdfEditor = new PdfEditor();
     private AiEngine aiEngine = new AiEngine();
     private InternalApi internalApi = new InternalApi();
+    private Cluster cluster = new Cluster();
 
     @Bean
     public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
@@ -254,6 +256,106 @@ public class ApplicationProperties {
         private int longRunningTimeoutSeconds = 600;
     }
 
+    /**
+     * Cluster backplane configuration. All keys live under the top-level {@code cluster.*} prefix
+     * (e.g. env var {@code CLUSTER_ENABLED}). The master switch is {@link #enabled} and defaults to
+     * off; when off the in-process backplane is wired and no other cluster keys are required.
+     */
+    @Data
+    public static class Cluster {
+
+        /** Master switch. When {@code false} (default) the in-process backplane is wired. */
+        private boolean enabled = false;
+
+        /** Backplane implementation selector. Valid values: {@code inprocess} | {@code valkey}. */
+        private String backplane = "inprocess";
+
+        /**
+         * Transient cluster job-artifact store selector. Valid values: {@code local} | {@code s3}.
+         *
+         * <p>This is distinct from {@code storage.provider}, which selects the backend for
+         * persistent user-uploaded files. The two switches exist because the user-facing storage
+         * feature is optional ({@code storage.enabled=false} is common) but every multi-node
+         * cluster still needs a shared artifact store to serve cross-node downloads. Both
+         * implementations share credentials from {@code storage.s3.*} when set to {@code s3}.
+         */
+        private String artifactStore = "local";
+
+        private Valkey valkey = new Valkey();
+        private Node node = new Node();
+
+        private transient String cachedNodeId;
+
+        public NodeRole resolvedRole() {
+            if (node == null || node.getRole() == null) {
+                return NodeRole.BOTH;
+            }
+            String value = node.getRole().trim().toUpperCase(Locale.ROOT);
+            try {
+                return NodeRole.valueOf(value);
+            } catch (IllegalArgumentException ex) {
+                return NodeRole.BOTH;
+            }
+        }
+
+        public synchronized String resolvedNodeId() {
+            if (node != null && node.getId() != null && !node.getId().isBlank()) {
+                return node.getId();
+            }
+            if (cachedNodeId == null) {
+                cachedNodeId = UUID.randomUUID().toString();
+            }
+            return cachedNodeId;
+        }
+
+        public enum NodeRole {
+            WEB,
+            WORKER,
+            BOTH
+        }
+
+        @Data
+        public static class Valkey {
+            /**
+             * {@code redis://host:6379} or {@code rediss://...} for TLS. Required when cluster mode
+             * is on and backplane is valkey.
+             */
+            private String url = "";
+
+            private Tls tls = new Tls();
+
+            @Data
+            public static class Tls {
+                /**
+                 * When {@code true}, skip Valkey/Redis TLS certificate verification (dev/test
+                 * only). Leave {@code false} in production.
+                 */
+                private boolean skipCertVerification = false;
+            }
+        }
+
+        @Data
+        public static class Node {
+            /** Optional explicit node id. Blank = auto-generated UUID at startup. */
+            private String id = "";
+
+            /** {@code web} | {@code worker} | {@code both}. */
+            private String role = "both";
+
+            /**
+             * Internal cluster address advertised in the instance registry (host:port). Blank =
+             * derived at startup.
+             */
+            private String internalAddress = "";
+
+            /** {@code http} | {@code https} - scheme used when peers call this node. */
+            private String scheme = "http";
+
+            /** Heartbeat publish interval for the instance registry, in milliseconds. */
+            private long heartbeatIntervalMs = 5000;
+        }
+    }
+
     /**
      * HTTP timeouts for loopback calls to internal Stirling API endpoints, used by the AI workflow
      * executor and the pipeline processor. A bounded read timeout prevents a hung tool (e.g. an

app/common/src/main/java/stirling/software/common/service/FileStorage.java

@@ -1,15 +1,13 @@
 package stirling.software.common.service;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.UUID;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
 
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
@@ -18,9 +16,11 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.common.cluster.FileStore;
+
 /**
  * Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
- * system to handle file references.
+ * system to handle file references. Disk I/O is delegated to the injected {@link FileStore} bean.
  */
 @Service
 @RequiredArgsConstructor
@@ -30,251 +30,150 @@ public class FileStorage {
     /** Holds the result of a stream-to-disk store operation: the file ID and the bytes written. */
     public record StoredFile(String fileId, long size) {}
 
-    @Value("${stirling.tempDir:/tmp/stirling-files}")
-    private String tempDirPath;
-
     private final FileOrUploadService fileOrUploadService;
+    private final FileStore fileStore;
 
-    /**
-     * Store a file and return its unique ID
-     *
-     * @param file The file to store
-     * @return The unique ID assigned to the file
-     * @throws IOException If there is an error storing the file
-     */
     public String storeFile(MultipartFile file) throws IOException {
-        String fileId = generateFileId();
-        Path filePath = getFilePath(fileId);
-
-        // Ensure the directory exists
-        Files.createDirectories(filePath.getParent());
-
-        // Transfer the file to the storage location
-        file.transferTo(filePath.toFile());
-
-        log.debug("Stored file with ID: {}", fileId);
-        return fileId;
-    }
-
-    /**
-     * Store a byte array as a file and return its unique ID
-     *
-     * @param bytes The byte array to store
-     * @param originalName The original name of the file (for extension)
-     * @return The unique ID assigned to the file
-     * @throws IOException If there is an error storing the file
-     */
-    public String storeBytes(byte[] bytes, String originalName) throws IOException {
-        String fileId = generateFileId();
-        Path filePath = getFilePath(fileId);
-
-        // Ensure the directory exists
-        Files.createDirectories(filePath.getParent());
-
-        // Write the bytes to the file
-        Files.write(filePath, bytes);
-
-        log.debug("Stored byte array with ID: {}", fileId);
-        return fileId;
-    }
-
-    /**
-     * Retrieve a file by its ID as a MultipartFile
-     *
-     * @param fileId The ID of the file to retrieve
-     * @return The file as a MultipartFile
-     * @throws IOException If the file doesn't exist or can't be read
-     */
-    public MultipartFile retrieveFile(String fileId) throws IOException {
-        Path filePath = getFilePath(fileId);
-
-        if (!Files.exists(filePath)) {
-            throw new IOException("File not found with ID: " + fileId);
+        // Fast path: when Spring buffered the multipart to disk (typical for large uploads), the
+        // backing Resource exposes a real File. Hand the Path to the FileStore so it can do a
+        // file-to-file copy (Linux sendfile, no copy through Java heap) rather than streaming
+        // the bytes through an 8K buffer. Falls back to the InputStream path for in-memory
+        // multiparts, exotic Resource impls, and anything that does not back onto a File.
+        Resource res;
+        try {
+            res = file.getResource();
+        } catch (RuntimeException ignored) {
+            res = null;
         }
+        if (res != null && res.isFile()) {
+            try {
+                FileStore.Stored stored =
+                        fileStore.store(res.getFile().toPath(), file.getOriginalFilename());
+                log.debug("Stored file with ID: {} (fast path)", stored.fileId());
+                return stored.fileId();
+            } catch (IOException ex) {
+                // Some Resource impls advertise isFile()=true but throw on getFile(); fall through.
+                log.debug("Resource fast path failed, falling back to stream copy", ex);
+            }
+        }
+        try (InputStream in = file.getInputStream()) {
+            FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename());
+            log.debug("Stored file with ID: {}", stored.fileId());
+            return stored.fileId();
+        }
+    }
 
-        byte[] fileData = Files.readAllBytes(filePath);
+    public String storeBytes(byte[] bytes, String originalName) throws IOException {
+        FileStore.Stored stored = fileStore.store(new ByteArrayInputStream(bytes), originalName);
+        log.debug("Stored byte array with ID: {}", stored.fileId());
+        return stored.fileId();
+    }
+
+    public MultipartFile retrieveFile(String fileId) throws IOException {
+        byte[] fileData = fileStore.retrieveBytes(fileId);
         return fileOrUploadService.toMockMultipartFile(fileId, fileData);
     }
 
-    /**
-     * Retrieve a file by its ID as a byte array
-     *
-     * @param fileId The ID of the file to retrieve
-     * @return The file as a byte array
-     * @throws IOException If the file doesn't exist or can't be read
-     */
     public byte[] retrieveBytes(String fileId) throws IOException {
-        Path filePath = getFilePath(fileId);
-
-        if (!Files.exists(filePath)) {
-            throw new IOException("File not found with ID: " + fileId);
-        }
-
-        return Files.readAllBytes(filePath);
+        return fileStore.retrieveBytes(fileId);
     }
 
-    /**
-     * Retrieve a file by its ID as a streaming InputStream. The caller is responsible for closing
-     * the returned stream.
-     *
-     * @param fileId The ID of the file to retrieve
-     * @return A buffered InputStream for the file
-     * @throws IOException If the file doesn't exist or can't be read
-     */
     public InputStream retrieveInputStream(String fileId) throws IOException {
-        Path filePath = getFilePath(fileId);
-        // Let Files.newInputStream throw NoSuchFileException naturally — avoids TOCTOU race
-        // between exists-check and open when another thread may delete concurrently.
-        return new BufferedInputStream(Files.newInputStream(filePath));
+        return fileStore.retrieve(fileId);
     }
 
-    /**
-     * Store data from an InputStream as a file and return its unique ID and byte count. Streams
-     * directly to disk without buffering the entire content in heap.
-     *
-     * @param inputStream The input stream to read from
-     * @param originalName The original name of the file (unused, kept for API symmetry)
-     * @return A {@link StoredFile} containing the file ID and the number of bytes written
-     * @throws IOException If there is an error storing the file
-     */
     public StoredFile storeInputStream(InputStream inputStream, String originalName)
             throws IOException {
-        String fileId = generateFileId();
-        Path filePath = getFilePath(fileId);
-        Files.createDirectories(filePath.getParent());
-        long size = Files.copy(inputStream, filePath);
-        log.debug("Stored input stream with ID: {}", fileId);
-        return new StoredFile(fileId, size);
+        FileStore.Stored stored = fileStore.store(inputStream, originalName);
+        log.debug("Stored input stream with ID: {}", stored.fileId());
+        return new StoredFile(stored.fileId(), stored.size());
     }
 
     public String storeFromStreamingBody(StreamingResponseBody body, String originalName)
             throws IOException {
-        String fileId = generateFileId();
-        Path filePath = getFilePath(fileId);
-        Files.createDirectories(filePath.getParent());
-        boolean success = false;
-        try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(filePath))) {
-            body.writeTo(os);
-            success = true;
-        } finally {
-            if (!success) {
+        // Hold Throwable not IOException: an unchecked failure (NPE, IllegalState, OOM, etc.)
+        // from the body writer would otherwise close the pipe with EOF and the consumer would
+        // return a truncated file with no error surfaced to the caller.
+        AtomicReference<Throwable> bodyError = new AtomicReference<>();
+        try (PipedOutputStream out = new PipedOutputStream();
+                PipedInputStream in = new PipedInputStream(out, 8192)) {
+            var executor = Executors.newSingleThreadExecutor(Thread.ofVirtual().factory());
+            java.util.concurrent.Future<?> task = null;
+            try {
+                task =
+                        executor.submit(
+                                () -> {
+                                    try {
+                                        body.writeTo(out);
+                                    } catch (Throwable ex) {
+                                        bodyError.set(ex);
+                                    } finally {
+                                        try {
+                                            out.close();
+                                        } catch (IOException ignored) {
+                                            // closed on the consumer side too
+                                        }
+                                    }
+                                });
+                FileStore.Stored stored = fileStore.store(in, originalName);
+                Throwable writerErr = bodyError.get();
+                if (writerErr != null) {
+                    // Body failed mid-write: the FileStore persisted a truncated entry.
+                    // Best-effort delete so we don't leak partial files; never let cleanup
+                    // mask the original writer error.
+                    try {
+                        fileStore.delete(stored.fileId());
+                    } catch (RuntimeException cleanupEx) {
+                        log.warn(
+                                "Failed to delete partial file {} after writer error: {}",
+                                stored.fileId(),
+                                cleanupEx.getMessage());
+                    }
+                    if (writerErr instanceof IOException ioe) {
+                        throw ioe;
+                    }
+                    throw new IOException(
+                            "StreamingResponseBody writer failed: " + writerErr.getMessage(),
+                            writerErr);
+                }
+                log.debug("Stored StreamingResponseBody with ID: {}", stored.fileId());
+                return stored.fileId();
+            } finally {
+                // Interrupt and join the writer task: shutdown() alone returns immediately and a
+                // failed store leaves the writer running, leaking a thread per failed upload.
+                if (task != null && !task.isDone()) {
+                    task.cancel(true);
+                }
+                executor.shutdown();
                 try {
-                    Files.deleteIfExists(filePath);
-                } catch (IOException cleanupEx) {
-                    log.warn(
-                            "Failed to clean up partial file {} after store failure",
-                            filePath,
-                            cleanupEx);
+                    if (!executor.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS)) {
+                        executor.shutdownNow();
+                    }
+                } catch (InterruptedException ie) {
+                    executor.shutdownNow();
+                    Thread.currentThread().interrupt();
                 }
             }
         }
-        log.debug("Stored StreamingResponseBody with ID: {}", fileId);
-        return fileId;
     }
 
-    /**
-     * Persist a {@link Resource} body to disk, returning the generated file ID. Used by the async
-     * job pipeline to capture {@code ResponseEntity<Resource>} results produced by controllers.
-     */
     public String storeFromResource(Resource resource, String originalName) throws IOException {
-        String fileId = generateFileId();
-        Path filePath = getFilePath(fileId);
-        Files.createDirectories(filePath.getParent());
-        boolean success = false;
         try (InputStream in = resource.getInputStream()) {
-            Files.copy(in, filePath);
-            success = true;
-        } finally {
-            if (!success) {
-                try {
-                    Files.deleteIfExists(filePath);
-                } catch (IOException cleanupEx) {
-                    log.warn(
-                            "Failed to clean up partial file {} after store failure",
-                            filePath,
-                            cleanupEx);
-                }
-            }
+            FileStore.Stored stored = fileStore.store(in, originalName);
+            log.debug("Stored Resource with ID: {}", stored.fileId());
+            return stored.fileId();
         }
-        log.debug("Stored Resource with ID: {}", fileId);
-        return fileId;
     }
 
-    /**
-     * Delete a file by its ID
-     *
-     * @param fileId The ID of the file to delete
-     * @return true if the file was deleted, false otherwise
-     */
     public boolean deleteFile(String fileId) {
-        try {
-            Path filePath = getFilePath(fileId);
-            return Files.deleteIfExists(filePath);
-        } catch (IOException e) {
-            log.error("Error deleting file with ID: {}", fileId, e);
-            return false;
-        }
+        return fileStore.delete(fileId);
     }
 
-    /**
-     * Check if a file exists by its ID
-     *
-     * @param fileId The ID of the file to check
-     * @return true if the file exists, false otherwise
-     */
     public boolean fileExists(String fileId) {
-        Path filePath = getFilePath(fileId);
-        return Files.exists(filePath);
+        return fileStore.exists(fileId);
     }
 
-    /**
-     * Get the size of a file by its ID without loading the content into memory
-     *
-     * @param fileId The ID of the file
-     * @return The size of the file in bytes
-     * @throws IOException If the file doesn't exist or can't be read
-     */
     public long getFileSize(String fileId) throws IOException {
-        Path filePath = getFilePath(fileId);
-
-        if (!Files.exists(filePath)) {
-            throw new IOException("File not found with ID: " + fileId);
-        }
-
-        return Files.size(filePath);
-    }
-
-    /**
-     * Get the path for a file ID
-     *
-     * @param fileId The ID of the file
-     * @return The path to the file
-     * @throws IllegalArgumentException if fileId contains path traversal characters or resolves
-     *     outside base directory
-     */
-    private Path getFilePath(String fileId) {
-        // Validate fileId to prevent path traversal
-        if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
-            throw new IllegalArgumentException("Invalid file ID");
-        }
-
-        Path basePath = Path.of(tempDirPath).normalize().toAbsolutePath();
-        Path resolvedPath = basePath.resolve(fileId).normalize();
-
-        // Ensure resolved path is within the base directory
-        if (!resolvedPath.startsWith(basePath)) {
-            throw new IllegalArgumentException("File ID resolves to an invalid path");
-        }
-
-        return resolvedPath;
-    }
-
-    /**
-     * Generate a unique file ID
-     *
-     * @return A unique file ID
-     */
-    private String generateFileId() {
-        return UUID.randomUUID().toString();
+        return fileStore.size(fileId);
     }
 }

app/common/src/main/java/stirling/software/common/service/TaskManager.java

@@ -3,9 +3,13 @@ package stirling.software.common.service;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Duration;
+import java.time.Instant;
 import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -16,6 +20,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Service;
@@ -26,6 +31,10 @@ import jakarta.annotation.PreDestroy;
 
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.JobStoreEntry.JobState;
 import stirling.software.common.model.job.JobResult;
 import stirling.software.common.model.job.JobStats;
 import stirling.software.common.model.job.ResultFile;
@@ -40,20 +49,20 @@ public class TaskManager {
     private int jobResultExpiryMinutes = 30;
 
     private final FileStorage fileStorage;
+    private final JobStore jobStore;
+    private final ClusterBackplane clusterBackplane;
     private final ScheduledExecutorService cleanupExecutor =
             Executors.newSingleThreadScheduledExecutor(
                     Thread.ofVirtual().name("task-cleanup-", 0).factory());
 
-    /** Initialize the task manager and start the cleanup scheduler */
-    public TaskManager(FileStorage fileStorage) {
+    @Autowired
+    public TaskManager(
+            FileStorage fileStorage, JobStore jobStore, ClusterBackplane clusterBackplane) {
         this.fileStorage = fileStorage;
+        this.jobStore = jobStore;
+        this.clusterBackplane = clusterBackplane;
 
-        // Schedule periodic cleanup of old job results
-        cleanupExecutor.scheduleAtFixedRate(
-                this::cleanupOldJobs,
-                10, // Initial delay
-                10, // Interval
-                TimeUnit.MINUTES);
+        cleanupExecutor.scheduleAtFixedRate(this::cleanupOldJobs, 10, 10, TimeUnit.MINUTES);
 
         log.debug(
                 "Task manager initialized with job result expiry of {} minutes",
@@ -66,7 +75,9 @@ public class TaskManager {
      * @param jobId The job ID
      */
     public void createTask(String jobId) {
-        jobResults.put(jobId, JobResult.createNew(jobId));
+        JobResult result = JobResult.createNew(jobId);
+        jobResults.put(jobId, result);
+        writeThrough(jobId, result);
         log.debug("Created task with job ID: {}", jobId);
     }
 
@@ -79,6 +90,7 @@ public class TaskManager {
     public void setResult(String jobId, Object result) {
         JobResult jobResult = getOrCreateJobResult(jobId);
         jobResult.completeWithResult(result);
+        writeThrough(jobId, jobResult);
         log.debug("Set result for job ID: {}", jobId);
     }
 
@@ -101,6 +113,7 @@ public class TaskManager {
                         extractZipToIndividualFiles(fileId, originalFileName);
                 if (!extractedFiles.isEmpty()) {
                     jobResult.completeWithFiles(extractedFiles);
+                    writeThrough(jobId, jobResult);
                     log.debug(
                             "Set multiple file results for job ID: {} with {} files extracted from"
                                     + " ZIP",
@@ -127,6 +140,7 @@ public class TaskManager {
                     "Failed to get file size for job {}: {}. Using size 0.", jobId, e.getMessage());
             jobResult.completeWithSingleFile(fileId, originalFileName, contentType, 0);
         }
+        writeThrough(jobId, jobResult);
     }
 
     /**
@@ -138,6 +152,7 @@ public class TaskManager {
     public void setMultipleFileResults(String jobId, List<ResultFile> resultFiles) {
         JobResult jobResult = getOrCreateJobResult(jobId);
         jobResult.completeWithFiles(resultFiles);
+        writeThrough(jobId, jobResult);
         log.debug(
                 "Set multiple file results for job ID: {} with {} files",
                 jobId,
@@ -153,6 +168,7 @@ public class TaskManager {
     public void setError(String jobId, String error) {
         JobResult jobResult = getOrCreateJobResult(jobId);
         jobResult.failWithError(error);
+        writeThrough(jobId, jobResult);
         log.debug("Set error for job ID: {}: {}", jobId, error);
     }
 
@@ -169,6 +185,7 @@ public class TaskManager {
             // If no result or error has been set, mark it as complete with an empty result
             jobResult.completeWithResult("Task completed successfully");
         }
+        writeThrough(jobId, jobResult);
         log.debug("Marked job ID: {} as complete", jobId);
     }
 
@@ -205,6 +222,7 @@ public class TaskManager {
         JobResult jobResult = jobResults.get(jobId);
         if (jobResult != null) {
             jobResult.addNote(note);
+            writeThrough(jobId, jobResult);
             log.debug("Added note to job ID: {}: {}", jobId, note);
             return true;
         }
@@ -295,8 +313,11 @@ public class TaskManager {
         return jobResults.computeIfAbsent(jobId, JobResult::createNew);
     }
 
-    /** Clean up old completed job results */
+    /** Clean up old completed job results. No-op in cluster mode; the backplane TTL owns expiry. */
     public void cleanupOldJobs() {
+        if (clusterBackplane != null && !clusterBackplane.shouldRunLocalCleanup()) {
+            return;
+        }
         LocalDateTime expiryThreshold =
                 LocalDateTime.now().minus(jobResultExpiryMinutes, ChronoUnit.MINUTES);
         int removedCount = 0;
@@ -315,6 +336,9 @@ public class TaskManager {
 
                     // Remove the job result
                     jobResults.remove(entry.getKey());
+                    if (jobStore != null) {
+                        jobStore.delete(entry.getKey());
+                    }
                     removedCount++;
                 }
             }
@@ -327,6 +351,53 @@ public class TaskManager {
         }
     }
 
+    /** Mirror the in-memory {@code JobResult} into the cluster-visible {@link JobStore}. */
+    private void writeThrough(String jobId, JobResult result) {
+        if (jobStore == null) {
+            return;
+        }
+        try {
+            jobStore.put(toEntry(jobId, result), Duration.ofMinutes(jobResultExpiryMinutes));
+        } catch (RuntimeException ex) {
+            log.warn("JobStore write-through failed for job {}: {}", jobId, ex.getMessage());
+        }
+    }
+
+    private JobStoreEntry toEntry(String jobId, JobResult result) {
+        JobState state;
+        if (result.isComplete()) {
+            state = result.getError() != null ? JobState.FAILED : JobState.COMPLETE;
+        } else {
+            state = JobState.PENDING;
+        }
+        Instant createdAt = toInstant(result.getCreatedAt());
+        Instant completedAt = toInstant(result.getCompletedAt());
+        List<String> fileIds = new ArrayList<>();
+        if (result.hasFiles()) {
+            for (ResultFile rf : result.getAllResultFiles()) {
+                fileIds.add(rf.getFileId());
+            }
+        }
+        Map<String, String> meta = new HashMap<>();
+        if (result.getNotes() != null && !result.getNotes().isEmpty()) {
+            meta.put("notesCount", Integer.toString(result.getNotes().size()));
+        }
+        String owningNodeId = clusterBackplane == null ? "local" : clusterBackplane.localNodeId();
+        return new JobStoreEntry(
+                jobId,
+                state,
+                owningNodeId,
+                createdAt,
+                completedAt,
+                result.getError(),
+                fileIds,
+                meta);
+    }
+
+    private Instant toInstant(LocalDateTime ldt) {
+        return ldt == null ? null : ldt.atZone(ZoneId.systemDefault()).toInstant();
+    }
+
     /** Shutdown the cleanup executor */
     @PreDestroy
     public void shutdown() {
@@ -370,7 +441,7 @@ public class TaskManager {
             while ((entry = zipIn.getNextEntry()) != null) {
                 if (!entry.isDirectory()) {
                     String contentType = determineContentType(entry.getName());
-                    // storeInputStream returns the fileId and byte count — no extra stat needed
+                    // storeInputStream returns the fileId and byte count - no extra stat needed
                     FileStorage.StoredFile stored =
                             fileStorage.storeInputStream(zipIn, entry.getName());
 
@@ -458,7 +529,8 @@ public class TaskManager {
     }
 
     /**
-     * Find the job key that owns a given file ID.
+     * Find the job key that owns a given file ID. Checks the local in-memory map first, then falls
+     * back to the cluster-visible {@link JobStore}.
      *
      * @param fileId file identifier to look up
      * @return scoped job key if found, otherwise null
@@ -474,6 +546,18 @@ public class TaskManager {
                 }
             }
         }
+        if (jobStore != null) {
+            // Propagate JobStore failures: returning null on a backplane outage would conflate
+            // "no such file" with "lookup unavailable" and the caller would respond 404 to a
+            // transient blip that should be retried. Let Spring's exception handler surface a
+            // 5xx so clients know to retry.
+            try {
+                return jobStore.findJobIdByFileId(fileId).orElse(null);
+            } catch (RuntimeException e) {
+                log.warn("JobStore findJobIdByFileId failed for {}: {}", fileId, e.getMessage());
+                throw e;
+            }
+        }
         return null;
     }
 }

app/common/src/test/java/stirling/software/common/architecture/ArchitectureTest.java

@@ -56,4 +56,17 @@ class ArchitectureTest {
                         .resideInAPackage("stirling.software.saas..");
         rule.check(commonClasses);
     }
+
+    @Test
+    void clusterInterfacesHaveNoImplementationDependencies() {
+        ArchRule rule =
+                noClasses()
+                        .that()
+                        .resideInAPackage("stirling.software.common.cluster..")
+                        .should()
+                        .dependOnClassesThat()
+                        .resideInAnyPackage(
+                                "stirling.software.proprietary..", "stirling.software.saas..");
+        rule.check(commonClasses);
+    }
 }

app/common/src/test/java/stirling/software/common/cluster/BackplaneContractCompilationTest.java

@@ -0,0 +1,51 @@
+package stirling.software.common.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+class BackplaneContractCompilationTest {
+
+    @Test
+    void jobStoreEntryRecordRoundTrips() {
+        Instant now = Instant.now();
+        JobStoreEntry entry =
+                new JobStoreEntry(
+                        "job-1",
+                        JobStoreEntry.JobState.PENDING,
+                        "node-a",
+                        now,
+                        null,
+                        null,
+                        List.of("file-1"),
+                        Map.of("k", "v"));
+        assertEquals("job-1", entry.jobId());
+        assertEquals(JobStoreEntry.JobState.PENDING, entry.state());
+        assertEquals("node-a", entry.owningNodeId());
+        assertEquals(now, entry.createdAt());
+        assertEquals(List.of("file-1"), entry.fileIds());
+        assertEquals("v", entry.resultMeta().get("k"));
+    }
+
+    @Test
+    void clusterNodeRecordRoundTrips() {
+        Instant heartbeat = Instant.now();
+        ClusterNode node = new ClusterNode("node-a", "10.0.0.1:8080", heartbeat, "BOTH");
+        assertEquals("node-a", node.nodeId());
+        assertEquals("10.0.0.1:8080", node.internalAddress());
+        assertEquals(heartbeat, node.lastHeartbeat());
+        assertEquals("BOTH", node.role());
+    }
+
+    @Test
+    void rateLimitDecisionRecordRoundTrips() {
+        RateLimitStore.RateLimitDecision d = new RateLimitStore.RateLimitDecision(true, 7, 0L);
+        assertEquals(true, d.allowed());
+        assertEquals(7, d.remainingTokens());
+        assertEquals(0L, d.nanosToWaitForRefill());
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/ClusterConfigValidationTest.java

@@ -0,0 +1,65 @@
+package stirling.software.common.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.lang.reflect.Method;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.ApplicationProperties.Cluster;
+
+class ClusterConfigValidationTest {
+
+    @Test
+    void validationPassesWhenDisabled() {
+        ApplicationProperties props = new ApplicationProperties();
+        ClusterConfig config = new ClusterConfig(props);
+        assertDoesNotThrow(() -> invokeValidate(config));
+    }
+
+    @Test
+    void validationFailsWhenValkeyEnabledWithoutUrl() {
+        ApplicationProperties props = new ApplicationProperties();
+        Cluster cluster = props.getCluster();
+        cluster.setEnabled(true);
+        cluster.setBackplane("valkey");
+        ClusterConfig config = new ClusterConfig(props);
+        assertThrows(IllegalStateException.class, () -> invokeValidate(config));
+    }
+
+    @Test
+    void validationPassesWhenValkeyEnabledWithUrl() {
+        ApplicationProperties props = new ApplicationProperties();
+        Cluster cluster = props.getCluster();
+        cluster.setEnabled(true);
+        cluster.setBackplane("valkey");
+        cluster.getValkey().setUrl("redis://localhost:6379");
+        ClusterConfig config = new ClusterConfig(props);
+        assertDoesNotThrow(() -> invokeValidate(config));
+    }
+
+    @Test
+    void validationPassesWhenInProcessEnabled() {
+        ApplicationProperties props = new ApplicationProperties();
+        Cluster cluster = props.getCluster();
+        cluster.setEnabled(true);
+        cluster.setBackplane("inprocess");
+        ClusterConfig config = new ClusterConfig(props);
+        assertDoesNotThrow(() -> invokeValidate(config));
+    }
+
+    private void invokeValidate(ClusterConfig config) throws Exception {
+        Method m = ClusterConfig.class.getDeclaredMethod("validate");
+        m.setAccessible(true);
+        try {
+            m.invoke(config);
+        } catch (java.lang.reflect.InvocationTargetException ex) {
+            if (ex.getCause() instanceof RuntimeException re) {
+                throw re;
+            }
+            throw ex;
+        }
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/ClusterPropertiesTest.java

@@ -0,0 +1,62 @@
+package stirling.software.common.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.ApplicationProperties.Cluster;
+
+class ClusterPropertiesTest {
+
+    @Test
+    void defaultsAreDisabledAndInprocess() {
+        Cluster props = new ApplicationProperties().getCluster();
+        assertFalse(props.isEnabled());
+        assertEquals("inprocess", props.getBackplane());
+        assertEquals("local", props.getArtifactStore());
+        assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
+        assertEquals("", props.getValkey().getUrl());
+        assertFalse(props.getValkey().getTls().isSkipCertVerification());
+        assertEquals("both", props.getNode().getRole());
+        assertEquals("http", props.getNode().getScheme());
+        assertEquals(5000L, props.getNode().getHeartbeatIntervalMs());
+    }
+
+    @Test
+    void resolvedRoleParsesCaseInsensitively() {
+        Cluster props = new ApplicationProperties().getCluster();
+        props.getNode().setRole("WEB");
+        assertEquals(Cluster.NodeRole.WEB, props.resolvedRole());
+
+        props.getNode().setRole("web");
+        assertEquals(Cluster.NodeRole.WEB, props.resolvedRole());
+
+        props.getNode().setRole("Worker");
+        assertEquals(Cluster.NodeRole.WORKER, props.resolvedRole());
+
+        props.getNode().setRole("garbage");
+        assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
+
+        props.getNode().setRole(null);
+        assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
+    }
+
+    @Test
+    void resolvedNodeIdIsStableAcrossCalls() {
+        Cluster props = new ApplicationProperties().getCluster();
+        String first = props.resolvedNodeId();
+        String second = props.resolvedNodeId();
+        assertNotNull(first);
+        assertEquals(first, second);
+    }
+
+    @Test
+    void resolvedNodeIdHonoursExplicitId() {
+        Cluster props = new ApplicationProperties().getCluster();
+        props.getNode().setId("abc");
+        assertEquals("abc", props.resolvedNodeId());
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/InProcessConfigurationConditionalTest.java

@@ -0,0 +1,90 @@
+package stirling.software.common.cluster;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import stirling.software.common.cluster.inprocess.InProcessClusterConfiguration;
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Verifies the {@link InProcessClusterConfiguration} conditional wiring: in-process beans wire when
+ * cluster mode is off or {@code backplane=inprocess}, and are skipped when {@code
+ * backplane=valkey}.
+ */
+class InProcessConfigurationConditionalTest {
+
+    private final ApplicationContextRunner runner =
+            new ApplicationContextRunner()
+                    .withConfiguration(
+                            org.springframework.boot.autoconfigure.AutoConfigurations.of(
+                                    PropertyPlaceholderAutoConfiguration.class))
+                    .withUserConfiguration(
+                            TestAppPropertiesConfig.class,
+                            ClusterConfig.class,
+                            InProcessClusterConfiguration.class);
+
+    @Test
+    void inProcessBeansWireWhenClusterDisabled() {
+        runner.run(
+                context ->
+                        assertThat(context)
+                                .hasNotFailed()
+                                .hasSingleBean(ClusterBackplane.class)
+                                .hasSingleBean(JobStore.class)
+                                .hasSingleBean(RateLimitStore.class)
+                                .hasSingleBean(DistributedLock.class)
+                                .hasSingleBean(KeyValueCache.class)
+                                .hasSingleBean(InstanceRegistry.class));
+    }
+
+    @Test
+    void inProcessBeansWireWhenEnabledWithInProcessBackplane() {
+        runner.withPropertyValues("cluster.enabled=true", "cluster.backplane=inprocess")
+                .run(
+                        context ->
+                                assertThat(context)
+                                        .hasNotFailed()
+                                        .hasSingleBean(ClusterBackplane.class)
+                                        .hasSingleBean(JobStore.class)
+                                        .hasSingleBean(RateLimitStore.class)
+                                        .hasSingleBean(DistributedLock.class)
+                                        .hasSingleBean(KeyValueCache.class)
+                                        .hasSingleBean(InstanceRegistry.class));
+    }
+
+    @Test
+    void inProcessBeansSkippedWhenEnabledWithDistributedBackplane() {
+        runner.withPropertyValues(
+                        "cluster.enabled=true",
+                        "cluster.backplane=valkey",
+                        "cluster.valkey.url=redis://localhost:6379")
+                .run(
+                        context ->
+                                assertThat(context)
+                                        .hasNotFailed()
+                                        .doesNotHaveBean(ClusterBackplane.class)
+                                        .doesNotHaveBean(JobStore.class)
+                                        .doesNotHaveBean(RateLimitStore.class)
+                                        .doesNotHaveBean(DistributedLock.class)
+                                        .doesNotHaveBean(KeyValueCache.class)
+                                        .doesNotHaveBean(InstanceRegistry.class));
+    }
+
+    /**
+     * Hand-rolled {@link ApplicationProperties} bean: the production class loads YAML at startup
+     * via a {@code @PostConstruct} hook that isn't appropriate for the slice runner, so we wire a
+     * defaults-only instance here.
+     */
+    @Configuration
+    static class TestAppPropertiesConfig {
+        @Bean
+        ApplicationProperties applicationProperties() {
+            return new ApplicationProperties();
+        }
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java

@@ -0,0 +1,167 @@
+package stirling.software.common.cluster.inprocess;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.cluster.DistributedLock;
+
+class InProcessDistributedLockTest {
+
+    @Test
+    void acquireReleaseAcquire() {
+        DistributedLock lock = new InProcessDistributedLock();
+        DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
+        h1.release();
+        assertTrue(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
+    }
+
+    @Test
+    void reentryFromSameThreadFails() {
+        DistributedLock lock = new InProcessDistributedLock();
+        DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
+        Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));
+        assertFalse(reentry.isPresent(), "in-process lock must be non-reentrant");
+        h1.release();
+        // After release, anyone can acquire again.
+        assertTrue(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
+    }
+
+    @Test
+    void secondAcquireFromAnotherThreadFails() throws InterruptedException {
+        DistributedLock lock = new InProcessDistributedLock();
+        DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
+
+        CountDownLatch done = new CountDownLatch(1);
+        AtomicBoolean acquired = new AtomicBoolean(true);
+        Thread t =
+                new Thread(
+                        () -> {
+                            Optional<DistributedLock.LockHandle> attempt =
+                                    lock.tryAcquire("k", Duration.ofSeconds(30));
+                            acquired.set(attempt.isPresent());
+                            attempt.ifPresent(DistributedLock.LockHandle::release);
+                            done.countDown();
+                        });
+        t.start();
+        assertTrue(done.await(2, TimeUnit.SECONDS));
+        assertFalse(acquired.get());
+        h1.release();
+    }
+
+    @Test
+    void leaseExpiryAllowsTakeoverEvenWithoutRelease() throws InterruptedException {
+        // Acquire with a short lease, never call release, then try to acquire again after the
+        // lease has elapsed. Matches Redis SET-NX-EX semantics - the second caller gets the lock
+        // because the first lease auto-expired. 250ms lease + 350ms wait gives CI generous slack.
+        DistributedLock lock = new InProcessDistributedLock();
+        DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofMillis(250)).orElseThrow();
+        Thread.sleep(350);
+        Optional<DistributedLock.LockHandle> takeover =
+                lock.tryAcquire("k", Duration.ofSeconds(30));
+        assertTrue(
+                takeover.isPresent(),
+                "expired lease must release the lock so a new caller can take over");
+        // Calling release() on the original handle after takeover must be a no-op (token check).
+        h1.release();
+        // The takeover holder is still the legitimate owner.
+        assertFalse(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
+        takeover.get().release();
+    }
+
+    @Test
+    void renewExtendsLease() throws InterruptedException {
+        // Acquire with a short lease, renew it before it expires, then verify the lock is still
+        // held past the original expiry point. 200ms initial + renew to 2s + wait 350ms.
+        DistributedLock lock = new InProcessDistributedLock();
+        DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofMillis(200)).orElseThrow();
+        assertTrue(h1.renew(Duration.ofSeconds(2)), "renew on a held lease must succeed");
+        Thread.sleep(350);
+        assertFalse(
+                lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent(),
+                "renew should have pushed expiry well past the original 200ms");
+        h1.release();
+    }
+
+    @Test
+    void renewAfterReleaseFails() {
+        DistributedLock lock = new InProcessDistributedLock();
+        DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
+        h1.release();
+        assertFalse(h1.renew(Duration.ofSeconds(30)), "renew on a released handle must fail");
+    }
+
+    /**
+     * Concurrency stress: many threads contending on the same key with each holder respecting the
+     * lease (hold &lt;&lt; lease). The lock behaves as a strict mutex in this regime so asserting
+     * mutual exclusion is meaningful. A separate test ({@link
+     * #leaseExpiryAllowsTakeoverEvenWithoutRelease}) covers the takeover-across-expiry branch,
+     * which legitimately allows two holders momentarily and is split-brain behaviour inherent to
+     * any lease-based lock.
+     */
+    @Test
+    void concurrentContentionPreservesMutualExclusion() throws InterruptedException {
+        DistributedLock lock = new InProcessDistributedLock();
+        int threads = 16;
+        int attemptsPerThread = 200;
+        // Lease far exceeds any plausible hold time, so the takeover branch never triggers in
+        // this test and the lock acts as a strict mutex.
+        Duration lease = Duration.ofSeconds(5);
+        java.util.concurrent.atomic.AtomicInteger concurrentHolders =
+                new java.util.concurrent.atomic.AtomicInteger();
+        java.util.concurrent.atomic.AtomicInteger maxConcurrent =
+                new java.util.concurrent.atomic.AtomicInteger();
+        java.util.concurrent.atomic.AtomicInteger acquires =
+                new java.util.concurrent.atomic.AtomicInteger();
+        java.util.concurrent.atomic.AtomicReference<Throwable> firstFailure =
+                new java.util.concurrent.atomic.AtomicReference<>();
+        CountDownLatch start = new CountDownLatch(1);
+        CountDownLatch done = new CountDownLatch(threads);
+
+        for (int i = 0; i < threads; i++) {
+            new Thread(
+                            () -> {
+                                try {
+                                    start.await();
+                                    for (int j = 0; j < attemptsPerThread; j++) {
+                                        Optional<DistributedLock.LockHandle> h =
+                                                lock.tryAcquire("hot", lease);
+                                        if (h.isPresent()) {
+                                            int now = concurrentHolders.incrementAndGet();
+                                            maxConcurrent.accumulateAndGet(now, Math::max);
+                                            acquires.incrementAndGet();
+                                            // Trivial critical section; well within lease.
+                                            concurrentHolders.decrementAndGet();
+                                            h.get().release();
+                                        }
+                                    }
+                                } catch (Throwable t) {
+                                    firstFailure.compareAndSet(null, t);
+                                } finally {
+                                    done.countDown();
+                                }
+                            },
+                            "lock-stress-" + i)
+                    .start();
+        }
+        start.countDown();
+        assertTrue(done.await(30, TimeUnit.SECONDS), "stress workers must finish in time");
+        org.junit.jupiter.api.Assertions.assertNull(firstFailure.get(), "no worker may throw");
+        org.junit.jupiter.api.Assertions.assertEquals(
+                1,
+                maxConcurrent.get(),
+                "mutual exclusion violated: more than one holder observed simultaneously");
+        assertTrue(
+                acquires.get() > 0,
+                "at least some acquires must succeed under contention (saw "
+                        + acquires.get()
+                        + ")");
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessInstanceRegistryTest.java

@@ -0,0 +1,28 @@
+package stirling.software.common.cluster.inprocess;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.cluster.ClusterNode;
+
+class InProcessInstanceRegistryTest {
+
+    @Test
+    void registerThenLookupAndActiveNodes() {
+        InProcessInstanceRegistry registry = new InProcessInstanceRegistry();
+        ClusterNode node = new ClusterNode("node-1", "127.0.0.1:8080", Instant.now(), "BOTH");
+        registry.register(node, Duration.ofSeconds(30));
+
+        assertTrue(registry.lookup("node-1").isPresent());
+        assertEquals("node-1", registry.lookup("node-1").get().nodeId());
+        assertEquals(1, registry.activeNodes().size());
+
+        registry.deregister("node-1");
+        assertTrue(registry.lookup("node-1").isEmpty());
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessJobStoreTest.java

@@ -0,0 +1,91 @@
+package stirling.software.common.cluster.inprocess;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.cluster.JobStoreEntry;
+
+class InProcessJobStoreTest {
+
+    private final InProcessJobStore store = new InProcessJobStore();
+
+    @Test
+    void putGetDeleteExistsRoundTrip() {
+        JobStoreEntry entry = entry("job-1");
+        store.put(entry, Duration.ofMinutes(30));
+
+        assertTrue(store.exists("job-1"));
+        assertEquals(entry, store.get("job-1").orElseThrow());
+
+        store.delete("job-1");
+        assertFalse(store.exists("job-1"));
+    }
+
+    @Test
+    void ttlExpiry() throws InterruptedException {
+        store.put(entry("job-2"), Duration.ofMillis(50));
+        Thread.sleep(100);
+        assertFalse(store.get("job-2").isPresent());
+    }
+
+    @Test
+    void purgeExpiredRemovesOnlyStaleEntries() throws InterruptedException {
+        store.put(entry("job-fresh"), Duration.ofMinutes(30));
+        store.put(entry("job-stale"), Duration.ofMillis(20));
+        Thread.sleep(80);
+
+        int removed = store.purgeExpired();
+        assertEquals(1, removed);
+        assertTrue(store.exists("job-fresh"));
+    }
+
+    @Test
+    void findJobIdByFileIdReturnsTheRightJob() {
+        store.put(
+                new JobStoreEntry(
+                        "job-a",
+                        JobStoreEntry.JobState.COMPLETE,
+                        "node-1",
+                        Instant.now(),
+                        Instant.now(),
+                        null,
+                        List.of("file-1", "file-2"),
+                        Map.of()),
+                Duration.ofMinutes(30));
+        store.put(
+                new JobStoreEntry(
+                        "job-b",
+                        JobStoreEntry.JobState.COMPLETE,
+                        "node-1",
+                        Instant.now(),
+                        Instant.now(),
+                        null,
+                        List.of("file-3"),
+                        Map.of()),
+                Duration.ofMinutes(30));
+
+        assertEquals("job-a", store.findJobIdByFileId("file-1").orElseThrow());
+        assertEquals("job-b", store.findJobIdByFileId("file-3").orElseThrow());
+        assertFalse(store.findJobIdByFileId("missing").isPresent());
+    }
+
+    private JobStoreEntry entry(String id) {
+        return new JobStoreEntry(
+                id,
+                JobStoreEntry.JobState.PENDING,
+                "node-1",
+                Instant.now(),
+                null,
+                null,
+                List.of(),
+                Map.of());
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessKeyValueCacheTest.java

@@ -0,0 +1,41 @@
+package stirling.software.common.cluster.inprocess;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import java.time.Duration;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.cluster.KeyValueCache;
+
+class InProcessKeyValueCacheTest {
+
+    @Test
+    void putGetEvict() {
+        KeyValueCache cache = new InProcessKeyValueCache();
+        cache.put("apikey", "a", "userA", Duration.ofMinutes(1));
+        assertEquals("userA", cache.get("apikey", "a").orElseThrow());
+
+        cache.evict("apikey", "a");
+        assertFalse(cache.get("apikey", "a").isPresent());
+    }
+
+    @Test
+    void ttlExpiry() throws InterruptedException {
+        KeyValueCache cache = new InProcessKeyValueCache();
+        cache.put("ns", "k", "v", Duration.ofMillis(40));
+        Thread.sleep(80);
+        assertFalse(cache.get("ns", "k").isPresent());
+    }
+
+    @Test
+    void evictNamespace() {
+        KeyValueCache cache = new InProcessKeyValueCache();
+        cache.put("ns", "a", "1", Duration.ofMinutes(1));
+        cache.put("ns", "b", "2", Duration.ofMinutes(1));
+        cache.evictNamespace("ns");
+        assertFalse(cache.get("ns", "a").isPresent());
+        assertFalse(cache.get("ns", "b").isPresent());
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessRateLimitStoreTest.java

@@ -0,0 +1,56 @@
+package stirling.software.common.cluster.inprocess;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.cluster.RateLimitStore;
+import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
+
+class InProcessRateLimitStoreTest {
+
+    @Test
+    void firstNConsumesAllowed() {
+        RateLimitStore store = new InProcessRateLimitStore();
+        for (int i = 0; i < 5; i++) {
+            assertTrue(store.tryConsume("k", 5, Duration.ofSeconds(60)).allowed(), "i=" + i);
+        }
+        assertFalse(store.tryConsume("k", 5, Duration.ofSeconds(60)).allowed());
+    }
+
+    @Test
+    void remainingTokensDecrements() {
+        RateLimitStore store = new InProcessRateLimitStore();
+        RateLimitDecision d1 = store.tryConsume("k", 5, Duration.ofSeconds(60));
+        RateLimitDecision d2 = store.tryConsume("k", 5, Duration.ofSeconds(60));
+        assertTrue(d1.allowed());
+        assertTrue(d2.allowed());
+        assertEquals(4, d1.remainingTokens());
+        assertEquals(3, d2.remainingTokens());
+    }
+
+    @Test
+    void refillRestoresTokens() throws InterruptedException {
+        RateLimitStore store = new InProcessRateLimitStore();
+        // Capacity 2 with smooth refill over 100 ms -> ~1 token per 50 ms.
+        for (int i = 0; i < 2; i++) {
+            assertTrue(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
+        }
+        assertFalse(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
+        Thread.sleep(150);
+        assertTrue(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
+    }
+
+    @Test
+    void deniedConsumeReportsWaitNanos() {
+        RateLimitStore store = new InProcessRateLimitStore();
+        assertTrue(store.tryConsume("wait", 1, Duration.ofSeconds(10)).allowed());
+        RateLimitDecision denied = store.tryConsume("wait", 1, Duration.ofSeconds(10));
+        assertFalse(denied.allowed());
+        assertTrue(denied.nanosToWaitForRefill() > 0L);
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/inprocess/LocalDiskFileStoreTest.java

@@ -0,0 +1,42 @@
+package stirling.software.common.cluster.inprocess;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import stirling.software.common.cluster.FileStore;
+
+class LocalDiskFileStoreTest {
+
+    @Test
+    void storeRetrieveSizeDeleteExistsRoundTrip(@TempDir Path dir) throws IOException {
+        LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
+        byte[] payload = "hello-bytes".getBytes();
+
+        FileStore.Stored stored = store.store(new ByteArrayInputStream(payload), "x.txt");
+        assertEquals(payload.length, stored.size());
+        assertTrue(store.exists(stored.fileId()));
+        assertEquals(payload.length, store.size(stored.fileId()));
+        assertArrayEquals(payload, store.retrieveBytes(stored.fileId()));
+
+        assertTrue(store.delete(stored.fileId()));
+        assertFalse(store.exists(stored.fileId()));
+    }
+
+    @Test
+    void traversalIdsAreRejected(@TempDir Path dir) {
+        LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
+        assertThrows(IllegalArgumentException.class, () -> store.resolve("../foo"));
+        assertThrows(IllegalArgumentException.class, () -> store.resolve("a/b"));
+        assertThrows(IllegalArgumentException.class, () -> store.resolve("a\\b"));
+    }
+}

app/common/src/test/java/stirling/software/common/service/FileStorageDelegationTest.java

@@ -0,0 +1,27 @@
+package stirling.software.common.service;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
+
+class FileStorageDelegationTest {
+
+    @Test
+    void storeBytesThenRetrieveBytesRoundTripsThroughFileStore(@TempDir Path tempDir)
+            throws IOException {
+        FileStorage fs =
+                new FileStorage(
+                        mock(FileOrUploadService.class),
+                        new LocalDiskFileStore(tempDir.toString()));
+        byte[] payload = "round-trip".getBytes();
+        String id = fs.storeBytes(payload, "x.bin");
+        assertArrayEquals(payload, fs.retrieveBytes(id));
+    }
+}

app/common/src/test/java/stirling/software/common/service/FileStorageTest.java

@@ -3,6 +3,7 @@ package stirling.software.common.service;
 import static org.junit.jupiter.api.Assertions.*;
 import static org.mockito.Mockito.*;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
@@ -13,29 +14,30 @@ import java.util.stream.Stream;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
-import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 import org.springframework.core.io.ByteArrayResource;
 import org.springframework.core.io.Resource;
 import org.springframework.http.MediaType;
-import org.springframework.test.util.ReflectionTestUtils;
 import org.springframework.web.multipart.MultipartFile;
 
+import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
+
 class FileStorageTest {
 
     @TempDir Path tempDir;
 
     @Mock private FileOrUploadService fileOrUploadService;
 
-    @InjectMocks private FileStorage fileStorage;
+    private FileStorage fileStorage;
 
     private MultipartFile mockFile;
 
     @BeforeEach
-    void setUp() {
+    void setUp() throws IOException {
         MockitoAnnotations.openMocks(this);
-        ReflectionTestUtils.setField(fileStorage, "tempDirPath", tempDir.toString());
+        fileStorage =
+                new FileStorage(fileOrUploadService, new LocalDiskFileStore(tempDir.toString()));
 
         // Create a mock MultipartFile
         mockFile = mock(MultipartFile.class);
@@ -47,17 +49,7 @@ class FileStorageTest {
     void testStoreFile() throws IOException {
         // Arrange
         byte[] fileContent = "Test PDF content".getBytes();
-        when(mockFile.getBytes()).thenReturn(fileContent);
-
-        // Set up mock to handle transferTo by writing the file
-        doAnswer(
-                        invocation -> {
-                            java.io.File file = invocation.getArgument(0);
-                            Files.write(file.toPath(), fileContent);
-                            return null;
-                        })
-                .when(mockFile)
-                .transferTo(any(java.io.File.class));
+        when(mockFile.getInputStream()).thenReturn(new ByteArrayInputStream(fileContent));
 
         // Act
         String fileId = fileStorage.storeFile(mockFile);
@@ -65,7 +57,7 @@ class FileStorageTest {
         // Assert
         assertNotNull(fileId);
         assertTrue(Files.exists(tempDir.resolve(fileId)));
-        verify(mockFile).transferTo(any(java.io.File.class));
+        assertArrayEquals(fileContent, Files.readAllBytes(tempDir.resolve(fileId)));
     }
 
     @Test
@@ -247,11 +239,11 @@ class FileStorageTest {
             filesBefore = s.count();
         }
 
-        // Act + Assert: IOException must propagate out — not be swallowed.
+        // Act + Assert: IOException must propagate out - not be swallowed.
         assertThrows(
                 IOException.class, () -> fileStorage.storeFromResource(flakyResource, "n.pdf"));
 
-        // Assert: no partial file lingers under the storage directory — the finally
+        // Assert: no partial file lingers under the storage directory - the finally
         // branch's deleteIfExists must have cleaned it up.
         long filesAfter;
         try (Stream<Path> s = Files.list(tempDir)) {

app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java

@@ -0,0 +1,120 @@
+package stirling.software.common.service;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.time.LocalDateTime;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.JobStoreEntry.JobState;
+import stirling.software.common.cluster.inprocess.InProcessClusterBackplane;
+import stirling.software.common.cluster.inprocess.InProcessJobStore;
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.job.JobResult;
+
+class TaskManagerJobStoreDelegationTest {
+
+    @Mock private FileStorage fileStorage;
+
+    private InProcessJobStore jobStore;
+    private ClusterBackplane backplane;
+    private TaskManager taskManager;
+
+    @BeforeEach
+    void setUp() {
+        MockitoAnnotations.openMocks(this);
+        jobStore = spy(new InProcessJobStore());
+        backplane = new InProcessClusterBackplane(new ApplicationProperties());
+        taskManager = new TaskManager(fileStorage, jobStore, backplane);
+        ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
+    }
+
+    @Test
+    void createTaskWritesPendingEntry() {
+        taskManager.createTask("job-1");
+        JobStoreEntry entry = jobStore.get("job-1").orElseThrow();
+        assertEquals(JobState.PENDING, entry.state());
+        assertEquals(backplane.localNodeId(), entry.owningNodeId());
+    }
+
+    @Test
+    void setCompleteFlipsToComplete() {
+        taskManager.createTask("job-2");
+        taskManager.setResult("job-2", "ok");
+        taskManager.setComplete("job-2");
+        JobStoreEntry entry = jobStore.get("job-2").orElseThrow();
+        assertEquals(JobState.COMPLETE, entry.state());
+    }
+
+    @Test
+    void setErrorFlipsToFailed() {
+        taskManager.createTask("job-3");
+        taskManager.setError("job-3", "boom");
+        JobStoreEntry entry = jobStore.get("job-3").orElseThrow();
+        assertEquals(JobState.FAILED, entry.state());
+        assertEquals("boom", entry.error());
+    }
+
+    @Test
+    void cleanupOldJobsIsNoopWhenBackplaneIsNotInProcess() {
+        ClusterBackplane mockedValkeyBackplane =
+                new ClusterBackplane() {
+                    @Override
+                    public boolean isHealthy() {
+                        return true;
+                    }
+
+                    @Override
+                    public String backplaneType() {
+                        return "valkey";
+                    }
+
+                    @Override
+                    public String localNodeId() {
+                        return "node-1";
+                    }
+
+                    @Override
+                    public boolean shouldRunLocalCleanup() {
+                        return false;
+                    }
+                };
+        TaskManager tm = new TaskManager(fileStorage, jobStore, mockedValkeyBackplane);
+        ReflectionTestUtils.setField(tm, "jobResultExpiryMinutes", 30);
+        tm.createTask("job-4");
+        tm.setComplete("job-4");
+        ageJobPastExpiry(tm, "job-4");
+        tm.cleanupOldJobs();
+        // cleanup must short-circuit before touching jobStore in cluster mode; the backplane
+        // TTL owns expiry there. If the gate fired correctly, delete is never called.
+        verify(jobStore, never()).delete(any());
+    }
+
+    @Test
+    void cleanupOldJobsDeletesFromJobStoreWhenBackplaneIsInProcess() {
+        taskManager.createTask("job-5");
+        taskManager.setComplete("job-5");
+        ageJobPastExpiry(taskManager, "job-5");
+        taskManager.cleanupOldJobs();
+        verify(jobStore).delete("job-5");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void ageJobPastExpiry(TaskManager tm, String jobId) {
+        var jobResults =
+                (java.util.Map<String, JobResult>) ReflectionTestUtils.getField(tm, "jobResults");
+        JobResult result = jobResults.get(jobId);
+        ReflectionTestUtils.setField(result, "completedAt", LocalDateTime.now().minusHours(2));
+        ReflectionTestUtils.setField(result, "complete", true);
+    }
+}

app/common/src/test/java/stirling/software/common/service/TaskManagerTest.java

@@ -1,20 +1,28 @@
 package stirling.software.common.service;
 
 import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.*;
 
 import java.time.LocalDateTime;
 import java.util.Map;
+import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 import org.springframework.http.MediaType;
 import org.springframework.test.util.ReflectionTestUtils;
 
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.JobStoreEntry.JobState;
 import stirling.software.common.model.job.JobResult;
 import stirling.software.common.model.job.JobStats;
 import stirling.software.common.model.job.ResultFile;
@@ -22,6 +30,8 @@ import stirling.software.common.model.job.ResultFile;
 class TaskManagerTest {
 
     @Mock private FileStorage fileStorage;
+    @Mock private JobStore jobStore;
+    @Mock private ClusterBackplane clusterBackplane;
 
     @InjectMocks private TaskManager taskManager;
 
@@ -30,6 +40,10 @@ class TaskManagerTest {
     @BeforeEach
     void setUp() {
         closeable = MockitoAnnotations.openMocks(this);
+        // Treat the backplane as in-process so cleanupOldJobs is not short-circuited.
+        lenient().when(clusterBackplane.backplaneType()).thenReturn("inprocess");
+        lenient().when(clusterBackplane.localNodeId()).thenReturn("test-node");
+        lenient().when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(true);
         ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
     }
 
@@ -270,6 +284,33 @@ class TaskManagerTest {
         verify(fileStorage).deleteFile("file-id");
     }
 
+    @Test
+    void testCleanupOldJobs_NoOpWhenBackplaneOwnsExpiry() {
+        // When the backplane reports it should NOT run local cleanup (e.g. a distributed
+        // backplane with its own TTL), the cleanup loop must leave local state untouched.
+        when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(false);
+
+        // Seed an old completed job that would normally be removed.
+        String oldJobId = "old-job-distributed";
+        taskManager.createTask(oldJobId);
+        JobResult oldJob = taskManager.getJobResult(oldJobId);
+        ReflectionTestUtils.setField(oldJob, "completedAt", LocalDateTime.now().minusHours(1));
+        ReflectionTestUtils.setField(oldJob, "complete", true);
+
+        Map<String, JobResult> jobResultsMap =
+                (Map<String, JobResult>) ReflectionTestUtils.getField(taskManager, "jobResults");
+        assertNotNull(jobResultsMap);
+        assertTrue(jobResultsMap.containsKey(oldJobId));
+
+        // Act
+        taskManager.cleanupOldJobs();
+
+        // Assert: nothing was removed locally, and no jobStore.delete was issued.
+        assertTrue(jobResultsMap.containsKey(oldJobId));
+        verify(jobStore, never()).delete(anyString());
+        verify(fileStorage, never()).deleteFile(anyString());
+    }
+
     @Test
     void testShutdown() {
         // This mainly tests that the shutdown method doesn't throw exceptions
@@ -310,4 +351,33 @@ class TaskManagerTest {
         // Assert
         assertFalse(result);
     }
+
+    @Test
+    void testWriteThroughOnUpdate() {
+        // Mutating calls must write through to the injected JobStore.
+        String jobId = "write-through-job";
+        taskManager.createTask(jobId);
+        taskManager.setResult(jobId, "done");
+
+        ArgumentCaptor<JobStoreEntry> captor = ArgumentCaptor.forClass(JobStoreEntry.class);
+        verify(jobStore, atLeast(2)).put(captor.capture(), any());
+
+        JobStoreEntry last = captor.getValue();
+        assertEquals(jobId, last.jobId());
+        assertEquals(JobState.COMPLETE, last.state());
+        assertEquals("test-node", last.owningNodeId());
+    }
+
+    @Test
+    void testFindJobKeyByFileId_FallsBackToJobStore() {
+        // When the file id is not in the local map, TaskManager delegates to JobStore.
+        String fileId = "remote-file-id";
+        String expectedJobKey = "remote-job-key";
+        when(jobStore.findJobIdByFileId(fileId)).thenReturn(Optional.of(expectedJobKey));
+
+        String actual = taskManager.findJobKeyByFileId(fileId);
+
+        assertEquals(expectedJobKey, actual);
+        verify(jobStore).findJobIdByFileId(fileId);
+    }
 }

app/core/src/main/resources/settings.yml.template

@@ -330,6 +330,22 @@ aiEngine:
   url: http://localhost:5001 # URL of the Python AI engine
   timeoutSeconds: 120 # Timeout in seconds for AI engine requests
 
+# Cluster configuration. NOT YET ENABLED - scaffolding for later work. Leave at defaults.
+cluster:
+  enabled: false # Master switch. 'false' (default) wires the in-process backplane and skips all cluster checks. Single-instance installs do not need to change anything here.
+  backplane: inprocess # Backplane implementation: 'inprocess' (single JVM only) or 'valkey' (multi-node via Valkey/Redis)
+  artifactStore: local # Transient cluster job-artifact backend: 'local' (per-node disk; single-node only) or 's3' (shared object store; required for multi-node). Distinct from 'storage.provider' which controls persistent user uploads - when both are 's3' they share the storage.s3.* credentials block. Multi-node deployments MUST set this to 's3'.
+  valkey:
+    url: "" # Valkey/Redis URL, e.g. 'redis://valkey:6379' or 'rediss://...' for TLS. Required when enabled=true and backplane=valkey.
+    tls:
+      skipCertVerification: false # set to 'true' to skip TLS certificate verification on Valkey connections (dev/test only)
+  node:
+    id: "" # Optional explicit node id. Blank = auto-generated UUID at startup.
+    role: both # 'web' (serves HTTP), 'worker' (runs jobs), or 'both' (default)
+    internalAddress: "" # host:port advertised in the instance registry for peer-to-peer cluster traffic. Blank = derived at startup.
+    scheme: http # 'http' or 'https' - scheme peers use to call this node's /internal/cluster/** endpoints
+    heartbeatIntervalMs: 5000 # Heartbeat publish interval for the instance registry (ms)
+
 pdfEditor:
   fallback-font: classpath:/static/fonts/NotoSans-Regular.ttf # Override to point at a custom fallback font
   cache:

deploy/aws/README.md

@@ -0,0 +1,235 @@
+# AWS deployment - Stirling-PDF clustered
+
+Three paths, ordered by ease-of-use:
+
+| Path | Best for | Time-to-live | Monthly cost (us-east-1) | Scales? |
+|---|---|---|---|---|
+| **1. CloudFormation one-click** | Enterprise self-host, no Kubernetes | ~15 min | ~$120-160 | Yes (ECS autoscaling) |
+| **2. Terraform module** | Your SaaS, env-promotable, GitOps | ~20 min first time | ~$120-160 | Yes (ECS autoscaling) |
+| **3. EC2 + Docker Compose** | ≤25 concurrent users, single VM, dead simple | ~5 min | ~$25-40 | No (one VM) |
+
+Already have a Kubernetes cluster (EKS)? Use the existing
+[`deploy/helm/stirling-pdf/`](../helm/stirling-pdf/) chart instead of any of
+these.
+
+## What each deploys
+
+All three deliver the same logical topology: **N Stirling app instances behind
+a load balancer, sharing a Valkey backplane and a Postgres database**, with
+`/internal/*` blocked at the LB.
+
+```
+            Internet
+                │
+                ▼
+       ┌────────────────┐
+       │  ALB / nginx   │  ←── blocks /internal/*
+       └────────────────┘
+            │       │
+            ▼       ▼
+       ┌────────────────┐
+       │  app-1   app-2 │  ←── CLUSTER_ENABLED=true
+       └────────────────┘       CLUSTER_BACKPLANE=valkey
+         │            │
+         ▼            ▼
+     Valkey       Postgres
+   (ElastiCache)    (RDS)
+```
+
+### Sticky sessions are required
+
+Every LB config in this repo (`ip_hash` in nginx, `lb_cookie` on the ALB,
+`affinity: cookie` on the k8s ingress) is pinning sessions deliberately, not as
+an optimisation. Result PDFs are written to the local disk of whichever node
+ran the job; without affinity a download has a ~50% chance of landing on a
+non-owner node and getting a 410 Gone. Cookie / IP affinity pins a returning
+client back to the owner pod. If you fork an LB config, keep the stickiness.
+
+**Heads-up on `ip_hash` (nginx Docker Compose path only):** `ip_hash` collapses
+every client sharing a source IP onto the same backend. Behind a corporate VPN,
+CGNAT, or a single egress NAT this means all of those users hammer one app
+container while the others sit idle. The Compose path is fine for small
+deployments (the doc above caps it at 25 concurrent users on one VM), but for
+diverse client populations move to cookie-based affinity (nginx-plus or the
+openresty sticky-cookie module) or use one of the managed LB paths above
+- the ALB / k8s Ingress configs in this directory already use cookie stickiness
+for exactly this reason.
+
+The only thing that changes between paths is **who manages Valkey and Postgres**:
+
+| Path | App tasks | Valkey | Postgres | LB |
+|---|---|---|---|---|
+| CloudFormation | ECS Fargate | ElastiCache for Valkey | RDS PostgreSQL | ALB |
+| Terraform | ECS Fargate | ElastiCache for Valkey | RDS PostgreSQL | ALB |
+| EC2 Compose | Docker on one EC2 | Docker container | Docker container | nginx (Docker) |
+
+## Path 1 - CloudFormation (recommended for enterprises)
+
+The template no longer accepts plaintext passwords as parameters - that pattern leaked
+secrets into the rendered stack template, CloudTrail events, and the template S3 bucket
+even with `NoEcho`. Operators pre-create the password secrets and pass their ARNs:
+
+```bash
+DB_SECRET_ARN=$(aws secretsmanager create-secret \
+  --name stirling/db-password \
+  --secret-string "$(openssl rand -base64 24)" \
+  --query ARN --output text)
+
+VALKEY_SECRET_ARN=$(aws secretsmanager create-secret \
+  --name stirling/valkey-auth \
+  --secret-string "$(openssl rand -hex 32)" \
+  --query ARN --output text)
+
+aws cloudformation deploy \
+  --stack-name stirling-pdf \
+  --template-file cloudformation/stirling-pdf-aws.yaml \
+  --capabilities CAPABILITY_NAMED_IAM \
+  --parameter-overrides \
+      DbSecretArn=$DB_SECRET_ARN \
+      ValkeyAuthSecretArn=$VALKEY_SECRET_ARN
+```
+
+The X-Engine-Auth token is generated *inside* Secrets Manager by CloudFormation - the
+operator never sees or supplies it. After deploy it can be retrieved with
+`aws secretsmanager get-secret-value --secret-id <stack-name>-engine-secret`.
+
+Or click-deploy via the AWS console: **Console → CloudFormation → Create stack
+→ Upload `cloudformation/stirling-pdf-aws.yaml` → fill the 2 ARN params**.
+
+The stack output `AppUrl` gives you the URL.
+
+**Knobs you can change in the parameters:**
+
+| Param | Default | Notes |
+|---|---|---|
+| `AppCount` | 2 | Initial Fargate task count; autoscaling can grow to 10 |
+| `AppCpu` | 1024 (1 vCPU) | Per task |
+| `AppMemory` | 4096 MiB | Per task |
+| `EnableAiEngine` | false | Turn on if using AI features |
+| `AppImage` | `stirlingtools/stirling-pdf:2.11.0` | Pinned to release. Swap for your own ECR URI or newer tag |
+| `EngineImage` | `stirlingtools/stirling-pdf-ai-engine:2.11.0` | Same |
+| `DbSecretArn` | (required) | ARN of pre-created Secrets Manager secret holding the Postgres password |
+| `ValkeyAuthSecretArn` | (required) | ARN of pre-created Secrets Manager secret holding the Valkey AUTH token |
+
+**Tear-down:** `aws cloudformation delete-stack --stack-name stirling-pdf`. RDS
+keeps a final snapshot for safety.
+
+## Path 2 - Terraform (recommended for your SaaS)
+
+```bash
+cd terraform
+cat > terraform.tfvars <<EOF
+name                 = "stirling-prod"
+region               = "us-east-1"
+app_count            = 3
+db_password          = "$(openssl rand -base64 24)"
+engine_shared_secret = "$(openssl rand -hex 32)"
+valkey_auth_token    = "$(openssl rand -hex 32)"
+EOF
+terraform init
+terraform apply
+```
+
+The ElastiCache replication group runs with TLS in flight (`rediss://`) and AUTH enabled.
+`valkey_auth_token` is required - the application connects with `REDIS_PASSWORD` populated
+from Secrets Manager so a compromised pod cannot sweep the keyspace.
+
+Tearing down: `terraform destroy`.
+
+The Terraform module is intentionally a **single file** to keep it easy to
+fork. For real production you'd split into `modules/{vpc,ecs,rds,elasticache,alb}`
+and have a separate `envs/{dev,staging,prod}/main.tf` referencing them. Both
+shapes work; the single file is the starting point.
+
+## Path 3 - EC2 + Docker Compose
+
+See [`quickstart/ec2-compose.md`](quickstart/ec2-compose.md).
+
+## EKS + Helm (for k8s shops)
+
+The Phase 1 implementation already ships a Helm chart:
+
+```bash
+# Assumes you already have an EKS cluster
+helm install stirling deploy/helm/stirling-pdf/ \
+  --set cluster.engineSharedSecret=$(openssl rand -hex 32) \
+  --set image.repository=<your-ecr-uri>/stirling-pdf
+```
+
+The chart includes a Valkey StatefulSet by default. To use ElastiCache instead:
+
+```bash
+helm install stirling deploy/helm/stirling-pdf/ \
+  --set cluster.valkey.bundled=false \
+  --set cluster.valkey.externalUrl=redis://your-elasticache:6379 \
+  ...
+```
+
+## Required AWS permissions (for whoever runs the deploy)
+
+Minimum set to apply the CloudFormation/Terraform:
+
+- `ec2:*`, `elasticloadbalancing:*` (VPC + ALB)
+- `ecs:*`, `iam:CreateRole`, `iam:AttachRolePolicy`, `iam:PutRolePolicy`,
+  `iam:PassRole`
+- `elasticache:*`
+- `rds:*`
+- `secretsmanager:*`
+- `logs:CreateLogGroup`, `logs:PutRetentionPolicy`
+- `application-autoscaling:*`
+- `cloudformation:*` (CFN only)
+
+Easiest: run as a user with `PowerUserAccess` for the initial bootstrap, then
+narrow down with IAM Access Analyzer after the stack is up.
+
+## Sizing rules of thumb
+
+| Concurrent users | App tasks | App size | Valkey | Postgres |
+|---|---|---|---|---|
+| ≤25 | 2 | 1 vCPU / 4 GB | cache.t4g.small | db.t4g.micro |
+| 25-100 | 3 | 2 vCPU / 4 GB | cache.t4g.small | db.t4g.small |
+| 100-500 | 4-6 | 2 vCPU / 8 GB | cache.t4g.medium | db.t4g.medium |
+| 500+ | 6+ (autoscale) | 2 vCPU / 8 GB | cache.r7g.large + replica | db.t4g.large + read replica |
+
+The defaults in the templates suit ≤25 users; bump `AppCount` /
+`InstanceClass` to grow.
+
+## What you still need to set up yourself
+
+These are not in the templates because they're customer-specific:
+
+- **Custom domain + HTTPS.** ALB listener on port 443 with an ACM cert (5-min
+  console wizard) + a Route 53 A-record alias to the ALB.
+- **Email (SES) for password reset / invitations.**
+- **OAuth/SAML identity provider.** Stirling supports Keycloak, Okta, Azure
+  AD, Google - config goes in `settings.yml`.
+- **Backups for Valkey.** Optional - ElastiCache supports daily snapshots; turn
+  on `SnapshotRetentionLimit` in the template if you want. Phase 1 state in
+  Valkey is short-TTL so most operators skip it.
+- **Frontend CDN.** Not required; the app serves static assets fine. If you
+  want CloudFront, point it at the ALB and cache `/static/*`.
+
+## Common questions
+
+**Why ECS Fargate, not EKS?** Fargate has zero cluster management - no
+control plane to maintain. EKS gets cheaper at scale but adds k8s ops. For
+most enterprises Fargate is the right default.
+
+**Why ElastiCache for Valkey and not just Redis?** Valkey is the Linux
+Foundation's BSD-licensed Redis fork; AWS ElastiCache has supported it natively
+since 2024. Same wire protocol. Stirling's `LettuceConnectionFactory` doesn't
+care which one - pick whichever your security team is happier with.
+
+**Why RDS Postgres and not Aurora?** Aurora costs ~3× more for the same TPS in
+this workload (mostly cold reads for user/team tables). Switch to Aurora if
+you need read replicas + DR; t4g.micro covers the small-tenant case.
+
+**What if I want to use my company's existing Postgres / Redis?** Run only
+the ECS part of the CloudFormation by setting `SPRING_DATASOURCE_URL` and
+`CLUSTER_VALKEY_URL` in the task definition to point at your
+existing endpoints. The template doesn't currently expose those as parameters
+- easy fork.
+
+**My ops team wants Pulumi / CDK / Crossplane.** All three speak the same
+underlying APIs the CloudFormation template uses. Translate from the YAML -
+the resource graph is identical.

deploy/aws/cloudformation/stirling-pdf-aws.yaml

@@ -0,0 +1,507 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: >
+  Stirling-PDF clustered deployment on AWS. Provisions an ECS Fargate service
+  (2+ app tasks) + ElastiCache for Valkey + RDS PostgreSQL + ALB. Single
+  CloudFormation stack - click Launch Stack and fill in 4 fields.
+
+  Cost estimate (us-east-1, default sizing): ~$120-160 / month.
+  Tear-down: delete the stack. Everything except the RDS final snapshot goes.
+
+Parameters:
+  AppImage:
+    Type: String
+    Default: stirlingtools/stirling-pdf:2.11.0
+    Description: Container image with the Phase 1 cluster code. Use your ECR URI for private builds. Pin to an exact version - never :latest in production.
+
+  EngineImage:
+    Type: String
+    Default: stirlingtools/stirling-pdf-ai-engine:2.11.0
+    Description: AI engine image. Leave default if you do not use AI features. Pin to an exact version - never :latest in production.
+
+  DbSecretArn:
+    Type: String
+    AllowedPattern: "^arn:aws:secretsmanager:[a-z0-9-]+:[0-9]+:secret:.+$"
+    Description: >
+      ARN of a Secrets Manager secret holding the Postgres password as plain SecretString.
+      Create: aws secretsmanager create-secret --name stirling/db-password --secret-string "$(openssl rand -base64 24)"
+      Passing the ARN keeps plaintext out of the rendered template and CloudTrail.
+
+  ValkeyAuthSecretArn:
+    Type: String
+    AllowedPattern: "^arn:aws:secretsmanager:[a-z0-9-]+:[0-9]+:secret:.+$"
+    Description: >
+      ARN of a Secrets Manager secret holding the ElastiCache AUTH token (16-128 chars).
+      Create: aws secretsmanager create-secret --name stirling/valkey-auth --secret-string "$(openssl rand -hex 32)"
+      Passing the ARN keeps plaintext out of the rendered template and CloudTrail.
+
+  AppCount:
+    Type: Number
+    Default: 2
+    MinValue: 2
+    MaxValue: 10
+    Description: Number of Stirling-PDF app tasks. Multi-instance requires at least 2.
+
+  AppCpu:
+    Type: Number
+    Default: 1024
+    AllowedValues: [512, 1024, 2048, 4096]
+    Description: Fargate CPU units per app task (1024 = 1 vCPU).
+
+  AppMemory:
+    Type: Number
+    Default: 4096
+    AllowedValues: [1024, 2048, 4096, 8192]
+    Description: Fargate memory MiB per app task.
+
+  EnableAiEngine:
+    Type: String
+    Default: "false"
+    AllowedValues: ["true", "false"]
+    Description: Set true to also run the AI engine service.
+
+Conditions:
+  WithAiEngine: !Equals [!Ref EnableAiEngine, "true"]
+
+Resources:
+
+  # ----- networking: use default VPC + its subnets to stay simple -----
+  Vpc:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: 10.42.0.0/16
+      EnableDnsHostnames: true
+      EnableDnsSupport: true
+      Tags: [{Key: Name, Value: !Sub "${AWS::StackName}-vpc"}]
+
+  Igw:
+    Type: AWS::EC2::InternetGateway
+  IgwAttach:
+    Type: AWS::EC2::VPCGatewayAttachment
+    Properties: {VpcId: !Ref Vpc, InternetGatewayId: !Ref Igw}
+
+  SubnetA:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref Vpc
+      AvailabilityZone: !Select [0, !GetAZs ""]
+      CidrBlock: 10.42.1.0/24
+      MapPublicIpOnLaunch: true
+  SubnetB:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref Vpc
+      AvailabilityZone: !Select [1, !GetAZs ""]
+      CidrBlock: 10.42.2.0/24
+      MapPublicIpOnLaunch: true
+
+  PublicRt:
+    Type: AWS::EC2::RouteTable
+    Properties: {VpcId: !Ref Vpc}
+  PublicRoute:
+    Type: AWS::EC2::Route
+    DependsOn: IgwAttach
+    Properties:
+      RouteTableId: !Ref PublicRt
+      DestinationCidrBlock: 0.0.0.0/0
+      GatewayId: !Ref Igw
+  RtAssocA:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties: {SubnetId: !Ref SubnetA, RouteTableId: !Ref PublicRt}
+  RtAssocB:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties: {SubnetId: !Ref SubnetB, RouteTableId: !Ref PublicRt}
+
+  # ----- security groups -----
+  AlbSg:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Public ALB
+      VpcId: !Ref Vpc
+      SecurityGroupIngress:
+        - {IpProtocol: tcp, FromPort: 80, ToPort: 80, CidrIp: 0.0.0.0/0}
+        - {IpProtocol: tcp, FromPort: 443, ToPort: 443, CidrIp: 0.0.0.0/0}
+
+  AppSg:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Stirling app tasks
+      VpcId: !Ref Vpc
+      SecurityGroupIngress:
+        - {IpProtocol: tcp, FromPort: 8080, ToPort: 8080, SourceSecurityGroupId: !Ref AlbSg}
+          - {IpProtocol: tcp, FromPort: 8080, ToPort: 8080, SourceSecurityGroupId: !GetAtt InternalSelfSg.GroupId}
+
+  InternalSelfSg:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Allow app tasks to reach each other internally
+      VpcId: !Ref Vpc
+
+  ValkeySg:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Valkey ElastiCache
+      VpcId: !Ref Vpc
+      SecurityGroupIngress:
+        - {IpProtocol: tcp, FromPort: 6379, ToPort: 6379, SourceSecurityGroupId: !Ref AppSg}
+
+  DbSg:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: PostgreSQL
+      VpcId: !Ref Vpc
+      SecurityGroupIngress:
+        - {IpProtocol: tcp, FromPort: 5432, ToPort: 5432, SourceSecurityGroupId: !Ref AppSg}
+
+  EngineLbSg:
+    Type: AWS::EC2::SecurityGroup
+    Condition: WithAiEngine
+    Properties:
+      GroupDescription: Internal ALB in front of the engine tier - only reachable from app tasks
+      VpcId: !Ref Vpc
+      SecurityGroupIngress:
+        - {IpProtocol: tcp, FromPort: 5001, ToPort: 5001, SourceSecurityGroupId: !Ref AppSg}
+
+  EngineSg:
+    Type: AWS::EC2::SecurityGroup
+    Condition: WithAiEngine
+    Properties:
+      GroupDescription: AI engine - only reachable from the internal engine LB
+      VpcId: !Ref Vpc
+      SecurityGroupIngress:
+        - {IpProtocol: tcp, FromPort: 5001, ToPort: 5001, SourceSecurityGroupId: !Ref EngineLbSg}
+
+  # ----- managed Valkey -----
+  ValkeySubnetGroup:
+    Type: AWS::ElastiCache::SubnetGroup
+    Properties:
+      Description: Stirling Valkey subnet group
+      SubnetIds: [!Ref SubnetA, !Ref SubnetB]
+
+  Valkey:
+    Type: AWS::ElastiCache::ReplicationGroup
+    Properties:
+      ReplicationGroupDescription: Stirling Valkey
+      Engine: valkey
+      EngineVersion: "8.0"
+      CacheNodeType: cache.t4g.small
+      NumCacheClusters: 1
+      AutomaticFailoverEnabled: false
+      CacheSubnetGroupName: !Ref ValkeySubnetGroup
+      SecurityGroupIds: [!Ref ValkeySg]
+      AtRestEncryptionEnabled: true
+      # TLS in flight required when AuthToken is set (ElastiCache enforces this).
+      TransitEncryptionEnabled: true
+      AuthToken: !Sub "{{resolve:secretsmanager:${ValkeyAuthSecretArn}}}"
+
+  # ----- managed Postgres -----
+  DbSubnetGroup:
+    Type: AWS::RDS::DBSubnetGroup
+    Properties:
+      DBSubnetGroupDescription: Stirling Postgres subnet group
+      SubnetIds: [!Ref SubnetA, !Ref SubnetB]
+
+  Postgres:
+    Type: AWS::RDS::DBInstance
+    DeletionPolicy: Snapshot
+    UpdateReplacePolicy: Snapshot
+    Properties:
+      DBInstanceIdentifier: !Sub "${AWS::StackName}-pg"
+      AllocatedStorage: 20
+      DBInstanceClass: db.t4g.micro
+      Engine: postgres
+      EngineVersion: "17.2"
+      MasterUsername: stirling
+      MasterUserPassword: !Sub "{{resolve:secretsmanager:${DbSecretArn}}}"
+      DBName: stirling
+      DBSubnetGroupName: !Ref DbSubnetGroup
+      VPCSecurityGroups: [!Ref DbSg]
+      StorageEncrypted: true
+      PubliclyAccessible: false
+      MultiAZ: false
+      BackupRetentionPeriod: 7
+
+  # ----- secrets -----
+  EngineSharedSecret:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Name: !Sub "${AWS::StackName}-engine-secret"
+      Description: X-Engine-Auth shared secret between app and AI engine. Generated by CFN, never passed in.
+      GenerateSecretString:
+        PasswordLength: 64
+        ExcludePunctuation: true
+
+
+  # ----- ECS cluster + roles -----
+  EcsCluster:
+    Type: AWS::ECS::Cluster
+    Properties:
+      ClusterName: !Sub "${AWS::StackName}-cluster"
+      ClusterSettings:
+        - {Name: containerInsights, Value: enabled}
+
+  TaskExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Statement:
+          - Effect: Allow
+            Principal: {Service: ecs-tasks.amazonaws.com}
+            Action: sts:AssumeRole
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
+      Policies:
+        - PolicyName: ReadSecrets
+          PolicyDocument:
+            Statement:
+              - Effect: Allow
+                Action: [secretsmanager:GetSecretValue]
+                Resource:
+                  - !Ref EngineSharedSecret
+                  - !Ref DbSecretArn
+                  - !Ref ValkeyAuthSecretArn
+
+  TaskLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/ecs/${AWS::StackName}"
+      RetentionInDays: 14
+
+  # ----- ALB -----
+  Alb:
+    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+    Properties:
+      Scheme: internet-facing
+      Subnets: [!Ref SubnetA, !Ref SubnetB]
+      SecurityGroups: [!Ref AlbSg]
+      Type: application
+
+  AppTg:
+    Type: AWS::ElasticLoadBalancingV2::TargetGroup
+    Properties:
+      VpcId: !Ref Vpc
+      Port: 8080
+      Protocol: HTTP
+      TargetType: ip
+      HealthCheckPath: /api/v1/info/status
+      HealthCheckIntervalSeconds: 30
+      HealthCheckTimeoutSeconds: 10
+      HealthyThresholdCount: 2
+      UnhealthyThresholdCount: 5
+      Matcher: {HttpCode: "200"}
+      # Sticky sessions required - see deploy/aws/README.md.
+      TargetGroupAttributes:
+        - Key: stickiness.enabled
+          Value: 'true'
+        - Key: stickiness.type
+          Value: lb_cookie
+        - Key: stickiness.lb_cookie.duration_seconds
+          Value: '86400'
+
+  AlbListener:
+    Type: AWS::ElasticLoadBalancingV2::Listener
+    Properties:
+      LoadBalancerArn: !Ref Alb
+      Port: 80
+      Protocol: HTTP
+      DefaultActions:
+        - Type: forward
+          TargetGroupArn: !Ref AppTg
+
+  AlbBlockInternalRule:
+    Type: AWS::ElasticLoadBalancingV2::ListenerRule
+    Properties:
+      ListenerArn: !Ref AlbListener
+      Priority: 1
+      Conditions:
+        - Field: path-pattern
+          Values: ["/internal/*"]
+      Actions:
+        - Type: fixed-response
+          FixedResponseConfig:
+            StatusCode: "404"
+            ContentType: text/plain
+            MessageBody: "Not Found"
+
+  # ----- ECS task definition for the Stirling app -----
+  AppTaskDef:
+    Type: AWS::ECS::TaskDefinition
+    Properties:
+      Family: !Sub "${AWS::StackName}-app"
+      Cpu: !Ref AppCpu
+      Memory: !Ref AppMemory
+      NetworkMode: awsvpc
+      RequiresCompatibilities: [FARGATE]
+      ExecutionRoleArn: !GetAtt TaskExecutionRole.Arn
+      ContainerDefinitions:
+        - Name: stirling
+          Image: !Ref AppImage
+          Essential: true
+          PortMappings:
+            - ContainerPort: 8080
+              Protocol: tcp
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group: !Ref TaskLogGroup
+              awslogs-region: !Ref AWS::Region
+              awslogs-stream-prefix: app
+          Environment:
+            - {Name: CLUSTER_ENABLED, Value: "true"}
+            - {Name: CLUSTER_BACKPLANE, Value: valkey}
+            # rediss:// = TLS. REDIS_PASSWORD injected separately via Secrets below.
+            - {Name: CLUSTER_VALKEY_URL, Value: !Sub "rediss://${Valkey.PrimaryEndPoint.Address}:6379"}
+            - {Name: SPRING_DATASOURCE_URL, Value: !Sub "jdbc:postgresql://${Postgres.Endpoint.Address}:5432/stirling"}
+            - {Name: SPRING_DATASOURCE_USERNAME, Value: stirling}
+            - {Name: DOCKER_ENABLE_SECURITY, Value: "true"}
+            - {Name: SYSTEM_DEFAULTLOCALE, Value: en-US}
+            - !If
+              - WithAiEngine
+              - {Name: AIENGINE_URL, Value: !Sub "http://${EngineAlb.DNSName}:5001"}
+              - !Ref AWS::NoValue
+          Secrets:
+            - {Name: CLUSTER_ENGINE_SHAREDSECRET, ValueFrom: !Ref EngineSharedSecret}
+            - {Name: SPRING_DATASOURCE_PASSWORD, ValueFrom: !Ref DbSecretArn}
+            - {Name: REDIS_PASSWORD, ValueFrom: !Ref ValkeyAuthSecretArn}
+
+  AppService:
+    Type: AWS::ECS::Service
+    DependsOn: AlbListener
+    Properties:
+      ServiceName: !Sub "${AWS::StackName}-app"
+      Cluster: !Ref EcsCluster
+      DesiredCount: !Ref AppCount
+      LaunchType: FARGATE
+      TaskDefinition: !Ref AppTaskDef
+      # Grace period covers Spring Boot warm-up + Valkey handshake (~60-90s total).
+      HealthCheckGracePeriodSeconds: 120
+      DeploymentConfiguration:
+        MinimumHealthyPercent: 50
+        MaximumPercent: 200
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: ENABLED
+          Subnets: [!Ref SubnetA, !Ref SubnetB]
+          SecurityGroups: [!Ref AppSg, !Ref InternalSelfSg]
+      LoadBalancers:
+        - ContainerName: stirling
+          ContainerPort: 8080
+          TargetGroupArn: !Ref AppTg
+
+  # ----- optional AI engine service -----
+  EngineAlb:
+    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+    Condition: WithAiEngine
+    Properties:
+      Scheme: internal
+      Subnets: [!Ref SubnetA, !Ref SubnetB]
+      SecurityGroups: [!Ref EngineLbSg]
+      Type: application
+
+  EngineTg:
+    Type: AWS::ElasticLoadBalancingV2::TargetGroup
+    Condition: WithAiEngine
+    Properties:
+      VpcId: !Ref Vpc
+      Port: 5001
+      Protocol: HTTP
+      TargetType: ip
+      HealthCheckPath: /health
+      HealthCheckIntervalSeconds: 30
+      HealthCheckTimeoutSeconds: 10
+      HealthyThresholdCount: 2
+      UnhealthyThresholdCount: 5
+      Matcher: {HttpCode: "200"}
+
+  EngineAlbListener:
+    Type: AWS::ElasticLoadBalancingV2::Listener
+    Condition: WithAiEngine
+    Properties:
+      LoadBalancerArn: !Ref EngineAlb
+      Port: 5001
+      Protocol: HTTP
+      DefaultActions:
+        - Type: forward
+          TargetGroupArn: !Ref EngineTg
+
+  EngineTaskDef:
+    Type: AWS::ECS::TaskDefinition
+    Condition: WithAiEngine
+    Properties:
+      Family: !Sub "${AWS::StackName}-engine"
+      Cpu: 1024
+      Memory: 2048
+      NetworkMode: awsvpc
+      RequiresCompatibilities: [FARGATE]
+      ExecutionRoleArn: !GetAtt TaskExecutionRole.Arn
+      ContainerDefinitions:
+        - Name: engine
+          Image: !Ref EngineImage
+          PortMappings: [{ContainerPort: 5001, Protocol: tcp}]
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group: !Ref TaskLogGroup
+              awslogs-region: !Ref AWS::Region
+              awslogs-stream-prefix: engine
+          Secrets:
+            - {Name: STIRLING_ENGINE_SHARED_SECRET, ValueFrom: !Ref EngineSharedSecret}
+
+  EngineService:
+    Type: AWS::ECS::Service
+    Condition: WithAiEngine
+    DependsOn: EngineAlbListener
+    Properties:
+      ServiceName: !Sub "${AWS::StackName}-engine"
+      Cluster: !Ref EcsCluster
+      DesiredCount: 1
+      LaunchType: FARGATE
+      TaskDefinition: !Ref EngineTaskDef
+      # Engine model-load warm-up can take 60-90s.
+      HealthCheckGracePeriodSeconds: 120
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: ENABLED
+          Subnets: [!Ref SubnetA, !Ref SubnetB]
+          SecurityGroups: [!Ref EngineSg]
+      LoadBalancers:
+        - ContainerName: engine
+          ContainerPort: 5001
+          TargetGroupArn: !Ref EngineTg
+
+  # ----- HPA-equivalent: target tracking on CPU -----
+  AppScalingTarget:
+    Type: AWS::ApplicationAutoScaling::ScalableTarget
+    Properties:
+      MaxCapacity: 10
+      MinCapacity: !Ref AppCount
+      ResourceId: !Sub "service/${EcsCluster}/${AppService.Name}"
+      ScalableDimension: ecs:service:DesiredCount
+      ServiceNamespace: ecs
+      RoleARN: !Sub "arn:aws:iam::${AWS::AccountId}:role/aws-service-role/ecs.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_ECSService"
+
+  AppScalingPolicy:
+    Type: AWS::ApplicationAutoScaling::ScalingPolicy
+    Properties:
+      PolicyName: !Sub "${AWS::StackName}-cpu-target"
+      PolicyType: TargetTrackingScaling
+      ScalingTargetId: !Ref AppScalingTarget
+      TargetTrackingScalingPolicyConfiguration:
+        TargetValue: 70
+        PredefinedMetricSpecification:
+          PredefinedMetricType: ECSServiceAverageCPUUtilization
+        ScaleInCooldown: 60
+        ScaleOutCooldown: 60
+
+Outputs:
+  AppUrl:
+    Description: Open this URL - Stirling-PDF cluster front door
+    Value: !Sub "http://${Alb.DNSName}/"
+  ClusterName:
+    Value: !Ref EcsCluster
+  ValkeyEndpoint:
+    Value: !GetAtt Valkey.PrimaryEndPoint.Address
+  PostgresEndpoint:
+    Value: !GetAtt Postgres.Endpoint.Address
+  StackTeardown:
+    Description: To delete everything (RDS keeps a final snapshot)
+    Value: !Sub "aws cloudformation delete-stack --stack-name ${AWS::StackName}"

deploy/aws/quickstart/ec2-compose.md

@@ -0,0 +1,108 @@
+# Single-EC2 Quickstart - "just give me Docker on a VM"
+
+The fastest possible AWS deployment when you don't want managed services. **Good
+for ≤25 concurrent users on a single beefy VM**. Past that, jump to the
+CloudFormation or Terraform option which scales horizontally.
+
+## What you get
+
+- One EC2 instance running 2 Stirling app containers, 1 Valkey, 1 Postgres,
+  1 nginx LB - exactly the validated `validation/compose.test.yml` topology
+- ~$25-40/month for a `t3.large`
+- 5-minute deploy
+
+## Steps
+
+### 1. Launch an EC2 instance
+
+- AMI: Amazon Linux 2023 (or Ubuntu 22.04 LTS)
+- Instance type: `t3.large` (2 vCPU, 8 GB RAM) minimum
+- Storage: 30 GB gp3
+- Security group: open `80/tcp` (and `22/tcp` for SSH)
+- IAM role: none needed
+
+### 2. SSH in and install Docker
+
+```bash
+sudo dnf install -y docker git
+sudo systemctl enable --now docker
+sudo usermod -aG docker ec2-user
+# new shell so the group takes effect
+exit
+```
+
+(On Ubuntu: `sudo apt install -y docker.io docker-compose-plugin git`.)
+
+### 3. Pull the compose stack
+
+```bash
+git clone https://github.com/Stirling-Tools/Stirling-PDF.git
+cd Stirling-PDF
+git checkout v2.11.0
+```
+
+### 4. Set secrets and start
+
+```bash
+export CLUSTER_ENGINE_SHAREDSECRET=$(openssl rand -hex 16)
+export STIRLING_VALKEY_PASSWORD=$(openssl rand -hex 16)
+export POSTGRES_PASSWORD=$(openssl rand -hex 16)
+docker compose -f docker/compose/docker-compose-cluster.yml up -d --build
+```
+
+Wait ~2 min for the apps to come up:
+
+```bash
+until curl -fsS http://localhost:8080/api/v1/info/status | grep -q UP; do sleep 5; done
+echo OK
+```
+
+### 5. Open it
+
+`http://<EC2 public IP>/`
+
+### 6. (Optional) HTTPS
+
+Slap Caddy or Traefik in front, or use AWS ALB pointing at the EC2 instance.
+Easiest is Caddy:
+
+```bash
+docker run -d --name caddy --restart unless-stopped \
+  -p 80:80 -p 443:443 \
+  -v caddy_data:/data \
+  -v $PWD/Caddyfile:/etc/caddy/Caddyfile \
+  caddy
+```
+
+with a tiny `Caddyfile`:
+
+```
+stirling.yourdomain.com {
+  reverse_proxy host.docker.internal:8080
+}
+```
+
+Caddy fetches a Let's Encrypt cert automatically.
+
+## Backups
+
+- **Postgres**: `docker exec stirling-postgres pg_dump -U stirling stirling > backup-$(date +%F).sql`
+  Cron this and ship to S3 with `aws s3 cp`.
+- **Valkey**: state is short-TTL (job status, rate-limit counters); no backup
+  needed - losing it on restart just means in-flight async jobs need re-running.
+
+## Tear-down
+
+`docker compose -f docker/compose/docker-compose-cluster.yml down -v` then
+terminate the EC2 instance.
+
+## Limits of this path
+
+- Single point of failure (one VM)
+- Manual scaling: edit `app-3`/`app-4` services into the compose file, restart
+- No autoscaling
+- No managed-service backups for Valkey
+- nginx LB runs on the same VM as the apps
+
+If any of these matter, use the CloudFormation or Terraform option in this
+directory instead.

deploy/aws/terraform/.terraform.lock.hcl

@@ -0,0 +1,25 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.100.0"
+  constraints = "~> 5.0"
+  hashes = [
+    "h1:H3mU/7URhP0uCRGK8jeQRKxx2XFzEqLiOq/L2Bbiaxs=",
+    "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644",
+    "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2",
+    "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274",
+    "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b",
+    "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862",
+    "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342",
+    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+    "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93",
+    "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2",
+    "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e",
+    "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421",
+    "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4",
+    "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9",
+    "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9",
+    "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70",
+  ]
+}

deploy/aws/terraform/.terraform/providers/registry.terraform.io/hashicorp/aws/5.100.0/windows_amd64/LICENSE.txt

@@ -0,0 +1,375 @@
+Copyright (c) 2017 HashiCorp, Inc.
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

deploy/aws/terraform/main.tf

@@ -0,0 +1,642 @@
+# Stirling-PDF on AWS - Terraform module (single-module starting point).
+# Production users typically split into modules/{vpc,ecs,rds,elasticache,alb}; this shape
+# is meant as a copy-and-fill-in template.
+
+terraform {
+  required_version = ">= 1.6"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.region
+}
+
+# ----- inputs -----
+variable "region" {
+  type    = string
+  default = "us-east-1"
+}
+
+variable "name" {
+  type    = string
+  default = "stirling"
+}
+
+variable "app_image" {
+  type    = string
+  # Never use :latest in production - breaks reproducible deploys and rollback.
+  default = "stirlingtools/stirling-pdf:2.11.0"
+}
+
+variable "engine_image" {
+  type    = string
+  default = "stirlingtools/stirling-pdf-ai-engine:2.11.0"
+}
+
+variable "app_count" {
+  type    = number
+  default = 2
+}
+
+variable "app_cpu" {
+  type    = number
+  default = 1024
+}
+
+variable "app_memory" {
+  type    = number
+  default = 4096
+}
+
+variable "engine_count" {
+  type    = number
+  default = 1
+}
+
+variable "enable_ai_engine" {
+  type    = bool
+  default = false
+}
+
+variable "db_password" {
+  type      = string
+  sensitive = true
+}
+
+variable "engine_shared_secret" {
+  type      = string
+  sensitive = true
+}
+
+variable "valkey_auth_token" {
+  # ElastiCache AUTH token (16-128 chars). Generate: openssl rand -hex 32
+  type      = string
+  sensitive = true
+  validation {
+    condition     = length(var.valkey_auth_token) >= 16 && length(var.valkey_auth_token) <= 128
+    error_message = "valkey_auth_token must be between 16 and 128 characters."
+  }
+}
+
+variable "vpc_cidr" {
+  type    = string
+  default = "10.42.0.0/16"
+}
+
+# ----- networking (slim VPC, two AZs) -----
+resource "aws_vpc" "main" {
+  cidr_block           = var.vpc_cidr
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+  tags = {
+    Name = "${var.name}-vpc"
+  }
+}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+resource "aws_subnet" "a" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = cidrsubnet(var.vpc_cidr, 8, 1)
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+}
+
+resource "aws_subnet" "b" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = cidrsubnet(var.vpc_cidr, 8, 2)
+  availability_zone       = data.aws_availability_zones.available.names[1]
+  map_public_ip_on_launch = true
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+}
+
+resource "aws_route_table_association" "a" {
+  subnet_id      = aws_subnet.a.id
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table_association" "b" {
+  subnet_id      = aws_subnet.b.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ----- security groups -----
+resource "aws_security_group" "alb" {
+  name        = "${var.name}-alb"
+  description = "Public ALB"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "app" {
+  name        = "${var.name}-app"
+  description = "Stirling app tasks"
+  vpc_id      = aws_vpc.main.id
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group_rule" "alb_to_app" {
+  type                     = "ingress"
+  from_port                = 8080
+  to_port                  = 8080
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.app.id
+  source_security_group_id = aws_security_group.alb.id
+}
+
+resource "aws_security_group_rule" "app_to_app" {
+  type                     = "ingress"
+  from_port                = 8080
+  to_port                  = 8080
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.app.id
+  source_security_group_id = aws_security_group.app.id
+}
+
+resource "aws_security_group" "valkey" {
+  name        = "${var.name}-valkey"
+  description = "Valkey ElastiCache"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    from_port       = 6379
+    to_port         = 6379
+    protocol        = "tcp"
+    security_groups = [aws_security_group.app.id]
+  }
+}
+
+resource "aws_security_group" "db" {
+  name        = "${var.name}-db"
+  description = "PostgreSQL"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    from_port       = 5432
+    to_port         = 5432
+    protocol        = "tcp"
+    security_groups = [aws_security_group.app.id]
+  }
+}
+
+resource "aws_security_group" "engine_lb" {
+  count       = var.enable_ai_engine ? 1 : 0
+  name        = "${var.name}-engine-lb"
+  description = "Internal ALB in front of engine tier - app tasks only"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    from_port       = 5001
+    to_port         = 5001
+    protocol        = "tcp"
+    security_groups = [aws_security_group.app.id]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "engine" {
+  count       = var.enable_ai_engine ? 1 : 0
+  name        = "${var.name}-engine"
+  description = "AI engine tasks - only reachable from the internal engine LB"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    from_port       = 5001
+    to_port         = 5001
+    protocol        = "tcp"
+    security_groups = [aws_security_group.engine_lb[0].id]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# ----- managed Valkey (ElastiCache for Valkey, GA since 2024) -----
+resource "aws_elasticache_subnet_group" "valkey" {
+  name       = "${var.name}-valkey"
+  subnet_ids = [aws_subnet.a.id, aws_subnet.b.id]
+}
+
+resource "aws_elasticache_replication_group" "valkey" {
+  replication_group_id       = "${var.name}-valkey"
+  description                = "Stirling Valkey"
+  engine                     = "valkey"
+  engine_version             = "8.0"
+  node_type                  = "cache.t4g.small"
+  num_cache_clusters         = 1
+  automatic_failover_enabled = false
+  subnet_group_name          = aws_elasticache_subnet_group.valkey.name
+  security_group_ids         = [aws_security_group.valkey.id]
+  at_rest_encryption_enabled = true
+  # TLS in flight required when auth_token is set (ElastiCache enforces this).
+  transit_encryption_enabled = true
+  auth_token                 = var.valkey_auth_token
+}
+
+# ----- managed Postgres -----
+resource "aws_db_subnet_group" "pg" {
+  name       = "${var.name}-pg"
+  subnet_ids = [aws_subnet.a.id, aws_subnet.b.id]
+}
+
+resource "aws_db_instance" "pg" {
+  identifier                = "${var.name}-pg"
+  engine                    = "postgres"
+  engine_version            = "17.2"
+  instance_class            = "db.t4g.micro"
+  allocated_storage         = 20
+  username                  = "stirling"
+  password                  = var.db_password
+  db_name                   = "stirling"
+  db_subnet_group_name      = aws_db_subnet_group.pg.name
+  vpc_security_group_ids    = [aws_security_group.db.id]
+  storage_encrypted         = true
+  skip_final_snapshot       = false
+  final_snapshot_identifier = "${var.name}-pg-final"
+  backup_retention_period   = 7
+}
+
+# ----- secrets -----
+resource "aws_secretsmanager_secret" "bundle" {
+  name = "${var.name}-secrets"
+}
+
+resource "aws_secretsmanager_secret_version" "bundle" {
+  secret_id = aws_secretsmanager_secret.bundle.id
+  secret_string = jsonencode({
+    engineSharedSecret = var.engine_shared_secret
+    dbPassword         = var.db_password
+    valkeyAuthToken    = var.valkey_auth_token
+  })
+}
+
+# ----- ECS cluster + IAM -----
+resource "aws_ecs_cluster" "main" {
+  name = "${var.name}-cluster"
+  setting {
+    name  = "containerInsights"
+    value = "enabled"
+  }
+}
+
+resource "aws_iam_role" "exec" {
+  name = "${var.name}-exec"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "ecs-tasks.amazonaws.com" }
+      Action    = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "exec_managed" {
+  role       = aws_iam_role.exec.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+}
+
+resource "aws_iam_role_policy" "exec_read_secrets" {
+  role = aws_iam_role.exec.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect   = "Allow"
+      Action   = ["secretsmanager:GetSecretValue"]
+      Resource = aws_secretsmanager_secret.bundle.arn
+    }]
+  })
+}
+
+resource "aws_cloudwatch_log_group" "ecs" {
+  name              = "/ecs/${var.name}"
+  retention_in_days = 14
+}
+
+# ----- ALB + target group + listener (blocks /internal/*) -----
+resource "aws_lb" "alb" {
+  name               = "${var.name}-alb"
+  load_balancer_type = "application"
+  subnets            = [aws_subnet.a.id, aws_subnet.b.id]
+  security_groups    = [aws_security_group.alb.id]
+}
+
+resource "aws_lb_target_group" "app" {
+  name        = "${var.name}-app-tg"
+  vpc_id      = aws_vpc.main.id
+  port        = 8080
+  protocol    = "HTTP"
+  target_type = "ip"
+
+  health_check {
+    path                = "/api/v1/info/status"
+    matcher             = "200"
+    interval            = 30
+    timeout             = 10
+    healthy_threshold   = 2
+    unhealthy_threshold = 5
+  }
+
+  # Sticky sessions required - see deploy/aws/README.md.
+  stickiness {
+    type            = "lb_cookie"
+    cookie_duration = 86400
+    enabled         = true
+  }
+}
+
+resource "aws_lb_listener" "http" {
+  load_balancer_arn = aws_lb.alb.arn
+  port              = 80
+  protocol          = "HTTP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.app.arn
+  }
+}
+
+resource "aws_lb_listener_rule" "block_internal" {
+  listener_arn = aws_lb_listener.http.arn
+  priority     = 1
+
+  condition {
+    path_pattern {
+      values = ["/internal/*"]
+    }
+  }
+
+  action {
+    type = "fixed-response"
+    fixed_response {
+      status_code  = "404"
+      content_type = "text/plain"
+      message_body = "Not Found"
+    }
+  }
+}
+
+# ----- app task definition + service -----
+resource "aws_ecs_task_definition" "app" {
+  family                   = "${var.name}-app"
+  cpu                      = var.app_cpu
+  memory                   = var.app_memory
+  network_mode             = "awsvpc"
+  requires_compatibilities = ["FARGATE"]
+  execution_role_arn       = aws_iam_role.exec.arn
+
+  container_definitions = jsonencode([{
+    name      = "stirling"
+    image     = var.app_image
+    essential = true
+    portMappings = [{
+      containerPort = 8080
+      protocol      = "tcp"
+    }]
+    logConfiguration = {
+      logDriver = "awslogs"
+      options = {
+        awslogs-group         = aws_cloudwatch_log_group.ecs.name
+        awslogs-region        = var.region
+        awslogs-stream-prefix = "app"
+      }
+    }
+    environment = concat([
+      { name = "CLUSTER_ENABLED", value = "true" },
+      { name = "CLUSTER_BACKPLANE", value = "valkey" },
+      # rediss:// = TLS. REDIS_PASSWORD injected separately via secrets below.
+      { name = "CLUSTER_VALKEY_URL", value = "rediss://${aws_elasticache_replication_group.valkey.primary_endpoint_address}:6379" },
+      { name = "SPRING_DATASOURCE_URL", value = "jdbc:postgresql://${aws_db_instance.pg.endpoint}/stirling" },
+      { name = "SPRING_DATASOURCE_USERNAME", value = "stirling" },
+      { name = "DOCKER_ENABLE_SECURITY", value = "true" },
+      ],
+      var.enable_ai_engine ? [
+        { name = "AIENGINE_URL", value = "http://${aws_lb.engine[0].dns_name}:5001" },
+      ] : []
+    )
+    secrets = [
+      { name = "CLUSTER_ENGINE_SHAREDSECRET", valueFrom = "${aws_secretsmanager_secret.bundle.arn}:engineSharedSecret::" },
+      { name = "SPRING_DATASOURCE_PASSWORD", valueFrom = "${aws_secretsmanager_secret.bundle.arn}:dbPassword::" },
+      { name = "REDIS_PASSWORD", valueFrom = "${aws_secretsmanager_secret.bundle.arn}:valkeyAuthToken::" },
+    ]
+  }])
+}
+
+resource "aws_ecs_service" "app" {
+  name                               = "${var.name}-app"
+  cluster                            = aws_ecs_cluster.main.id
+  task_definition                    = aws_ecs_task_definition.app.arn
+  desired_count                      = var.app_count
+  launch_type                        = "FARGATE"
+  deployment_minimum_healthy_percent = 50
+  deployment_maximum_percent         = 200
+  # Grace period covers Spring Boot warm-up + Valkey handshake (~60-90s total).
+  health_check_grace_period_seconds = 120
+
+  network_configuration {
+    subnets          = [aws_subnet.a.id, aws_subnet.b.id]
+    security_groups  = [aws_security_group.app.id]
+    assign_public_ip = true
+  }
+
+  load_balancer {
+    target_group_arn = aws_lb_target_group.app.arn
+    container_name   = "stirling"
+    container_port   = 8080
+  }
+
+  depends_on = [aws_lb_listener.http]
+}
+
+# ----- autoscaling -----
+resource "aws_appautoscaling_target" "app" {
+  max_capacity       = 10
+  min_capacity       = var.app_count
+  resource_id        = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.app.name}"
+  scalable_dimension = "ecs:service:DesiredCount"
+  service_namespace  = "ecs"
+}
+
+resource "aws_appautoscaling_policy" "cpu" {
+  name               = "${var.name}-cpu-target"
+  policy_type        = "TargetTrackingScaling"
+  resource_id        = aws_appautoscaling_target.app.resource_id
+  scalable_dimension = aws_appautoscaling_target.app.scalable_dimension
+  service_namespace  = aws_appautoscaling_target.app.service_namespace
+
+  target_tracking_scaling_policy_configuration {
+    target_value = 70
+    predefined_metric_specification {
+      predefined_metric_type = "ECSServiceAverageCPUUtilization"
+    }
+    scale_in_cooldown  = 60
+    scale_out_cooldown = 60
+  }
+}
+
+# ----- AI engine tier (internal LB + task def + service) -----
+resource "aws_lb" "engine" {
+  count              = var.enable_ai_engine ? 1 : 0
+  name               = "${var.name}-engine-alb"
+  internal           = true
+  load_balancer_type = "application"
+  subnets            = [aws_subnet.a.id, aws_subnet.b.id]
+  security_groups    = [aws_security_group.engine_lb[0].id]
+}
+
+resource "aws_lb_target_group" "engine" {
+  count       = var.enable_ai_engine ? 1 : 0
+  name        = "${var.name}-engine-tg"
+  vpc_id      = aws_vpc.main.id
+  port        = 5001
+  protocol    = "HTTP"
+  target_type = "ip"
+
+  health_check {
+    path                = "/health"
+    matcher             = "200"
+    interval            = 30
+    timeout             = 10
+    healthy_threshold   = 2
+    unhealthy_threshold = 5
+  }
+}
+
+resource "aws_lb_listener" "engine" {
+  count             = var.enable_ai_engine ? 1 : 0
+  load_balancer_arn = aws_lb.engine[0].arn
+  port              = 5001
+  protocol          = "HTTP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.engine[0].arn
+  }
+}
+
+resource "aws_ecs_task_definition" "engine" {
+  count                    = var.enable_ai_engine ? 1 : 0
+  family                   = "${var.name}-engine"
+  cpu                      = 1024
+  memory                   = 2048
+  network_mode             = "awsvpc"
+  requires_compatibilities = ["FARGATE"]
+  execution_role_arn       = aws_iam_role.exec.arn
+
+  container_definitions = jsonencode([{
+    name      = "engine"
+    image     = var.engine_image
+    essential = true
+    portMappings = [{
+      containerPort = 5001
+      protocol      = "tcp"
+    }]
+    logConfiguration = {
+      logDriver = "awslogs"
+      options = {
+        awslogs-group         = aws_cloudwatch_log_group.ecs.name
+        awslogs-region        = var.region
+        awslogs-stream-prefix = "engine"
+      }
+    }
+    secrets = [
+      { name = "STIRLING_ENGINE_SHARED_SECRET", valueFrom = "${aws_secretsmanager_secret.bundle.arn}:engineSharedSecret::" },
+    ]
+  }])
+}
+
+resource "aws_ecs_service" "engine" {
+  count           = var.enable_ai_engine ? 1 : 0
+  name            = "${var.name}-engine"
+  cluster         = aws_ecs_cluster.main.id
+  task_definition = aws_ecs_task_definition.engine[0].arn
+  desired_count   = var.engine_count
+  launch_type     = "FARGATE"
+  # Engine model-load warm-up can take 60-90s.
+  health_check_grace_period_seconds = 120
+
+  network_configuration {
+    subnets          = [aws_subnet.a.id, aws_subnet.b.id]
+    security_groups  = [aws_security_group.engine[0].id]
+    assign_public_ip = true
+  }
+
+  load_balancer {
+    target_group_arn = aws_lb_target_group.engine[0].arn
+    container_name   = "engine"
+    container_port   = 5001
+  }
+
+  depends_on = [aws_lb_listener.engine]
+}
+
+# ----- outputs -----
+output "app_url" {
+  value = "http://${aws_lb.alb.dns_name}/"
+}
+
+output "valkey_endpoint" {
+  value = aws_elasticache_replication_group.valkey.primary_endpoint_address
+}
+
+output "postgres_endpoint" {
+  value = aws_db_instance.pg.endpoint
+}
+
+output "cluster_name" {
+  value = aws_ecs_cluster.main.name
+}

deploy/helm/stirling-pdf/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: stirling-pdf
+description: Stirling-PDF clustered deployment (web + worker + AI engine + Valkey).
+type: application
+version: 0.1.0
+appVersion: "2.11.0"

deploy/helm/stirling-pdf/templates/NOTES.txt

@@ -0,0 +1,44 @@
+Stirling-PDF release "{{ .Release.Name }}" has been deployed.
+
+{{- if .Values.ingress.enabled }}
+Front door: https://{{ .Values.ingress.host }}/  (via Ingress class "{{ .Values.ingress.className }}")
+{{- if not .Values.ingress.tls.enabled }}
+
+WARNING: TLS is disabled on the Ingress (ingress.tls.enabled=false).
+  Cookies, JWTs, and API keys will cross the wire in plaintext. Acceptable
+  for dev / loopback only. Re-enable TLS and provide a secret named
+  "{{ .Values.ingress.tls.secretName }}" (cert-manager or kubectl create secret tls).
+{{- end }}
+{{- end }}
+
+{{- if and .Values.cluster.enabled (eq .Values.cluster.backplane "valkey") .Values.cluster.valkey.bundled }}
+
+WARNING: bundled Valkey is a single-replica StatefulSet (SPOF) - DEV / EVAL ONLY.
+  Production deployments MUST use an external HA Valkey/Redis: set
+    cluster.valkey.bundled=false
+    cluster.valkey.externalUrl=rediss://<managed-endpoint>:6379
+  Managed options: AWS ElastiCache for Valkey, GCP Memorystore, the Valkey
+  Operator with multi-replica + Sentinel, or any other HA Redis-protocol service.
+  See deploy/aws/README.md for the recommended managed paths.
+{{- end }}
+
+{{- if and .Values.cluster.enabled (not .Values.cluster.engineSharedSecret) }}
+
+INFO: cluster.engineSharedSecret was auto-generated and stored in the Secret
+  "{{ .Release.Name }}-cluster-secrets". It is stable across `helm upgrade`
+  (the chart reads the existing Secret on each render) but will regenerate on
+  `helm uninstall` + reinstall. To pin it, retrieve with:
+    kubectl get secret {{ .Release.Name }}-cluster-secrets \
+      -o jsonpath='{.data.engineSharedSecret}' | base64 -d
+  and store it in your secret manager / values file for future installs.
+{{- end }}
+
+{{- if and .Values.cluster.enabled (eq .Values.cluster.backplane "valkey") .Values.cluster.valkey.bundled (not .Values.cluster.valkey.password) }}
+
+INFO: cluster.valkey.password was auto-generated for the bundled Valkey and
+  stored in "{{ .Release.Name }}-cluster-secrets". Stable across upgrades but
+  regenerates on uninstall + reinstall (which would orphan the persisted
+  Valkey PVC). Pin it explicitly before going beyond eval/dev:
+    kubectl get secret {{ .Release.Name }}-cluster-secrets \
+      -o jsonpath='{.data.valkeyPassword}' | base64 -d
+{{- end }}

deploy/helm/stirling-pdf/templates/_helpers.tpl

@@ -0,0 +1,13 @@
+{{/* Resolve the Valkey URL - bundled vs. external bring-your-own.
+  Bundled path uses ${REDIS_PASSWORD} (shell expansion at Spring Boot startup) so the
+  password is never written into the rendered manifest or K8s events.
+*/}}
+{{- define "stirling-pdf.valkeyUrl" -}}
+{{- if and .Values.cluster.enabled .Values.cluster.valkey.bundled -}}
+redis://:${REDIS_PASSWORD}@{{ .Release.Name }}-valkey:6379
+{{- else if .Values.cluster.valkey.externalUrl -}}
+{{ .Values.cluster.valkey.externalUrl }}
+{{- else if .Values.cluster.enabled -}}
+{{ fail "cluster.valkey.bundled=false requires cluster.valkey.externalUrl to be set (e.g. rediss://user:pw@host:6379)" }}
+{{- end -}}
+{{- end -}}

deploy/helm/stirling-pdf/templates/engine-deployment.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-engine
+spec:
+  replicas: {{ .Values.engine.replicas }}
+  selector:
+    matchLabels:
+      app: stirling-pdf
+      role: engine
+  template:
+    metadata:
+      labels:
+        app: stirling-pdf
+        role: engine
+    spec:
+      containers:
+        - name: engine
+          image: {{ .Values.engine.image.repository }}:{{ .Values.engine.image.tag | default .Chart.AppVersion }}
+          env:
+            - name: STIRLING_ENGINE_SHARED_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-cluster-secrets
+                  key: engineSharedSecret
+          ports:
+            - containerPort: 5001
+          # Startup probe: 30 x 5s = 150s grace for model-load warm-up. Requires k8s >= 1.18.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 5001
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 5001
+            periodSeconds: 10
+            failureThreshold: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 5001
+            periodSeconds: 10
+            failureThreshold: 10
+          {{- with .Values.engine.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-engine
+spec:
+  selector:
+    app: stirling-pdf
+    role: engine
+  ports:
+    - port: 5001
+      targetPort: 5001

deploy/helm/stirling-pdf/templates/hpa.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.web.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ .Release.Name }}-web
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Release.Name }}-web
+  minReplicas: {{ .Values.web.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.web.autoscaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.web.autoscaling.targetCPUUtilizationPercentage }}
+{{- end }}
+---
+{{- if .Values.worker.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ .Release.Name }}-worker
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Release.Name }}-worker
+  minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }}
+{{- end }}

deploy/helm/stirling-pdf/templates/ingress.yaml

@@ -0,0 +1,45 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ .Release.Name }}
+  annotations:
+    # Cookie affinity is required - see deploy/aws/README.md "Sticky sessions are required".
+    nginx.ingress.kubernetes.io/affinity: "cookie"
+    nginx.ingress.kubernetes.io/affinity-mode: "persistent"
+    nginx.ingress.kubernetes.io/session-cookie-name: "STIRLING_NODE"
+    nginx.ingress.kubernetes.io/session-cookie-max-age: "86400"
+    # Match the JVM-side 2000MB multipart limit (nginx-ingress default 1MB would 413).
+    nginx.ingress.kubernetes.io/proxy-body-size: "2000m"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    # Defense-in-depth: /internal/* is never a public route.
+    nginx.ingress.kubernetes.io/server-snippet: |
+      location ^~ /internal/ { return 404; }
+spec:
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- if .Values.ingress.tls.enabled }}
+  {{- if not .Values.ingress.tls.secretName }}
+  {{- fail "ingress.tls.enabled=true requires ingress.tls.secretName to be set" }}
+  {{- end }}
+  tls:
+    - hosts:
+        {{- if .Values.ingress.tls.hosts }}
+        {{- toYaml .Values.ingress.tls.hosts | nindent 8 }}
+        {{- else }}
+        - {{ .Values.ingress.host }}
+        {{- end }}
+      secretName: {{ .Values.ingress.tls.secretName }}
+  {{- end }}
+  rules:
+    - host: {{ .Values.ingress.host }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ .Release.Name }}-web
+                port:
+                  number: 8080
+{{- end }}

deploy/helm/stirling-pdf/templates/secrets.yaml

@@ -0,0 +1,59 @@
+{{/*
+  Stable-secret pattern. Resolution order for each field:
+    1. Operator-provided value (.Values.*) wins.
+    2. Otherwise reuse what is already in the cluster Secret (lookup), so
+       `helm upgrade` does NOT churn the value on every run.
+    3. Otherwise generate a fresh 64-char random value (first install).
+
+  Caveats:
+  - `lookup` returns empty during `helm template` / `--dry-run=client`. That
+    is acceptable here: those modes are for inspection, not source of truth.
+    Real `helm install` / `helm upgrade` against a live API server see the
+    existing Secret.
+  - `helm uninstall` removes this Secret. A subsequent `helm install` with no
+    operator override will generate fresh values, and any persisted data tied
+    to the old Valkey password (PVC contents) will be unreadable.
+*/}}
+{{- $secretName := printf "%s-cluster-secrets" .Release.Name }}
+{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }}
+{{- $existingEngine := "" }}
+{{- $existingValkey := "" }}
+{{- if $existing }}
+{{-   if hasKey $existing.data "engineSharedSecret" }}
+{{-     $existingEngine = index $existing.data "engineSharedSecret" | b64dec }}
+{{-   end }}
+{{-   if hasKey $existing.data "valkeyPassword" }}
+{{-     $existingValkey = index $existing.data "valkeyPassword" | b64dec }}
+{{-   end }}
+{{- end }}
+
+{{- $engineSecret := "" }}
+{{- if .Values.cluster.engineSharedSecret }}
+{{-   $engineSecret = .Values.cluster.engineSharedSecret }}
+{{- else if $existingEngine }}
+{{-   $engineSecret = $existingEngine }}
+{{- else }}
+{{-   $engineSecret = randAlphaNum 64 }}
+{{- end }}
+
+{{- $needsBundledValkey := and .Values.cluster.enabled (eq .Values.cluster.backplane "valkey") .Values.cluster.valkey.bundled }}
+{{- $valkeyPassword := "" }}
+{{- if $needsBundledValkey }}
+{{-   if .Values.cluster.valkey.password }}
+{{-     $valkeyPassword = .Values.cluster.valkey.password }}
+{{-   else if $existingValkey }}
+{{-     $valkeyPassword = $existingValkey }}
+{{-   else }}
+{{-     $valkeyPassword = randAlphaNum 64 }}
+{{-   end }}
+{{- end }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+stringData:
+  engineSharedSecret: {{ $engineSecret | quote }}
+  {{- if $needsBundledValkey }}
+  valkeyPassword: {{ $valkeyPassword | quote }}
+  {{- end }}

deploy/helm/stirling-pdf/templates/valkey.yaml

@@ -0,0 +1,80 @@
+{{- if and .Values.cluster.enabled .Values.cluster.valkey.bundled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Release.Name }}-valkey
+spec:
+  serviceName: {{ .Release.Name }}-valkey
+  replicas: 1
+  selector:
+    matchLabels:
+      app: stirling-valkey
+  template:
+    metadata:
+      labels:
+        app: stirling-valkey
+    spec:
+      containers:
+        - name: valkey
+          image: valkey/valkey:8.0-alpine
+          command:
+            - sh
+            - -c
+            - "exec valkey-server --requirepass \"$REDIS_PASSWORD\" --maxmemory 256mb --maxmemory-policy allkeys-lru"
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-cluster-secrets
+                  key: valkeyPassword
+          ports:
+            - containerPort: 6379
+          readinessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - "valkey-cli -a \"$REDIS_PASSWORD\" --no-auth-warning ping"
+            initialDelaySeconds: 2
+            periodSeconds: 5
+          livenessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - "valkey-cli -a \"$REDIS_PASSWORD\" --no-auth-warning ping"
+            initialDelaySeconds: 15
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.cluster.valkey.resources | nindent 12 }}
+          {{- if .Values.cluster.valkey.persistence.enabled }}
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          {{- end }}
+  {{- if .Values.cluster.valkey.persistence.enabled }}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        {{- if .Values.cluster.valkey.persistence.storageClassName }}
+        storageClassName: {{ .Values.cluster.valkey.persistence.storageClassName }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.cluster.valkey.persistence.size }}
+  {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-valkey
+spec:
+  type: ClusterIP
+  selector:
+    app: stirling-valkey
+  ports:
+    - port: 6379
+      targetPort: 6379
+{{- end }}

deploy/helm/stirling-pdf/templates/web-deployment.yaml

@@ -0,0 +1,75 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-web
+  labels:
+    app: stirling-pdf
+    role: web
+spec:
+  replicas: {{ .Values.web.replicas }}
+  selector:
+    matchLabels:
+      app: stirling-pdf
+      role: web
+  template:
+    metadata:
+      labels:
+        app: stirling-pdf
+        role: web
+    spec:
+      containers:
+        - name: app
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: MODE
+              value: FRONTEND
+            - name: CLUSTER_ENABLED
+              value: {{ .Values.cluster.enabled | quote }}
+            - name: CLUSTER_BACKPLANE
+              value: {{ .Values.cluster.backplane }}
+            {{- if and .Values.cluster.enabled (eq .Values.cluster.backplane "valkey") .Values.cluster.valkey.bundled }}
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-cluster-secrets
+                  key: valkeyPassword
+            {{- end }}
+            - name: CLUSTER_VALKEY_URL
+              value: {{ include "stirling-pdf.valkeyUrl" . | quote }}
+            - name: CLUSTER_NODE_ROLE
+              value: web
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CLUSTER_ENGINE_SHAREDSECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-cluster-secrets
+                  key: engineSharedSecret
+          ports:
+            - containerPort: 8080
+          # Startup probe: 30 x 5s = 150s grace for Spring Boot + Valkey warm-up. Requires k8s >= 1.18.
+          startupProbe:
+            httpGet:
+              path: /api/v1/info/status
+              port: 8080
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /api/v1/info/status
+              port: 8080
+            periodSeconds: 10
+            failureThreshold: 10
+          livenessProbe:
+            httpGet:
+              path: /api/v1/info/status
+              port: 8080
+            periodSeconds: 10
+            failureThreshold: 10
+          {{- with .Values.web.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}

deploy/helm/stirling-pdf/templates/web-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-web
+spec:
+  selector:
+    app: stirling-pdf
+    role: web
+  ports:
+    - port: 8080
+      targetPort: 8080

deploy/helm/stirling-pdf/templates/worker-deployment.yaml

@@ -0,0 +1,75 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-worker
+  labels:
+    app: stirling-pdf
+    role: worker
+spec:
+  replicas: {{ .Values.worker.replicas }}
+  selector:
+    matchLabels:
+      app: stirling-pdf
+      role: worker
+  template:
+    metadata:
+      labels:
+        app: stirling-pdf
+        role: worker
+    spec:
+      containers:
+        - name: app
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: MODE
+              value: BACKEND
+            - name: CLUSTER_ENABLED
+              value: {{ .Values.cluster.enabled | quote }}
+            - name: CLUSTER_BACKPLANE
+              value: {{ .Values.cluster.backplane }}
+            {{- if and .Values.cluster.enabled (eq .Values.cluster.backplane "valkey") .Values.cluster.valkey.bundled }}
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-cluster-secrets
+                  key: valkeyPassword
+            {{- end }}
+            - name: CLUSTER_VALKEY_URL
+              value: {{ include "stirling-pdf.valkeyUrl" . | quote }}
+            - name: CLUSTER_NODE_ROLE
+              value: worker
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CLUSTER_ENGINE_SHAREDSECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-cluster-secrets
+                  key: engineSharedSecret
+          ports:
+            - containerPort: 8080
+          # Startup probe: 30 x 5s = 150s grace for Spring Boot + Valkey warm-up. Requires k8s >= 1.18.
+          startupProbe:
+            httpGet:
+              path: /api/v1/info/status
+              port: 8080
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /api/v1/info/status
+              port: 8080
+            periodSeconds: 10
+            failureThreshold: 10
+          livenessProbe:
+            httpGet:
+              path: /api/v1/info/status
+              port: 8080
+            periodSeconds: 10
+            failureThreshold: 10
+          {{- with .Values.worker.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}

deploy/helm/stirling-pdf/values.yaml

@@ -0,0 +1,91 @@
+# Stirling-PDF Helm chart - clustered install (web + worker + AI engine + Valkey behind Ingress).
+# Single-instance deploys can run the standard container directly without this chart.
+
+image:
+  repository: stirlingtools/stirling-pdf
+  # Leave blank to default to .Chart.AppVersion. Override per release.
+  tag: ""
+  pullPolicy: IfNotPresent
+
+engine:
+  image:
+    repository: stirlingtools/stirling-pdf-ai-engine
+    tag: ""
+  replicas: 2
+  resources:
+    requests:
+      cpu: 250m
+      memory: 512Mi
+    limits:
+      cpu: "2"
+      memory: 1Gi
+
+web:
+  replicas: 2
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+    maxReplicas: 10
+    targetCPUUtilizationPercentage: 70
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      cpu: "2"
+      memory: 2Gi
+
+worker:
+  replicas: 2
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+    maxReplicas: 10
+    targetCPUUtilizationPercentage: 70
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      cpu: "2"
+      memory: 2Gi
+
+cluster:
+  enabled: true
+  backplane: valkey
+  valkey:
+    # WARNING: bundled Valkey is a single-replica StatefulSet (SPOF) - DEV / EVAL ONLY.
+    # Production: set bundled=false and point externalUrl at HA Valkey/Redis.
+    # See deploy/aws/README.md for managed options (ElastiCache, Memorystore, etc.).
+    bundled: true
+    # Required when bundled=false. Use rediss:// for TLS.
+    externalUrl: ""
+    # Auto-generated on first install if empty (stable across upgrades via Secret lookup).
+    # Set explicitly via --set or values file when using your own secret manager.
+    password: ""
+    persistence:
+      enabled: true
+      size: 1Gi
+      storageClassName: ""
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 500m
+        memory: 384Mi
+  # Java <-> AI engine auth secret. Auto-generated if empty (see NOTES.txt for retrieval).
+  engineSharedSecret: ""
+
+ingress:
+  enabled: true
+  host: stirling.example.com
+  className: nginx
+  tls:
+    # WARNING: TLS MUST be on in production - cookies, JWTs, and API keys cross the wire.
+    # Provision the secret below (cert-manager, kubectl create secret tls, etc.) before helm install.
+    # tls.enabled=false is HTTP-only - dev/loopback ONLY.
+    enabled: true
+    secretName: stirling-pdf-tls
+    # Defaults to [ingress.host] when empty.
+    hosts: []

deploy/observability/grafana-cluster-dashboard.json

@@ -0,0 +1,175 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {"type": "grafana", "uid": "-- Grafana --"},
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Starter dashboard for Stirling-PDF cluster mode (plan §7). Add to your Grafana instance pointed at a Prometheus that is scraping /actuator/prometheus on each node. NOTE: requires PR3 (Valkey backplane impls + ClusterMetrics) - panels render 'No data' until that lands.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+      "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}},
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+      "id": 1,
+      "options": {"legend": {"displayMode": "table", "showLegend": true}},
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "stirling_cluster_jobs_inflight",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Jobs in flight (per node)",
+      "type": "timeseries",
+      "description": "Requires PR3 (Valkey backplane impls + ClusterMetrics) - import after PR3 lands."
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+      "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}},
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
+      "id": 2,
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "stirling_cluster_queue_depth",
+          "legendFormat": "{{lane}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Queue depth (per lane)",
+      "type": "timeseries",
+      "description": "Requires PR3 (Valkey backplane impls + ClusterMetrics) - import after PR3 lands."
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+      "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "s"}},
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
+      "id": 3,
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "histogram_quantile(0.5, sum(rate(stirling_cluster_job_wait_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "histogram_quantile(0.95, sum(rate(stirling_cluster_job_wait_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p95",
+          "refId": "B"
+        },
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "histogram_quantile(0.99, sum(rate(stirling_cluster_job_wait_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p99",
+          "refId": "C"
+        }
+      ],
+      "title": "Job wait time (p50/p95/p99)",
+      "type": "timeseries",
+      "description": "Requires PR3 (Valkey backplane impls + ClusterMetrics) - import after PR3 lands."
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+      "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "s"}},
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
+      "id": 4,
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "histogram_quantile(0.5, sum(rate(stirling_cluster_backplane_latency_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "histogram_quantile(0.95, sum(rate(stirling_cluster_backplane_latency_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p95",
+          "refId": "B"
+        }
+      ],
+      "title": "Backplane round-trip latency (Valkey p50/p95)",
+      "type": "timeseries",
+      "description": "Requires PR3 (Valkey backplane impls + ClusterMetrics) - import after PR3 lands."
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+      "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "ops"}},
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
+      "id": 5,
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "sum(rate(stirling_cluster_sticky_miss_total[5m]))",
+          "legendFormat": "sticky-session misses / sec",
+          "refId": "A"
+        }
+      ],
+      "title": "Sticky-session miss rate (high = LB affinity broken)",
+      "type": "timeseries",
+      "description": "Requires PR3 (Valkey backplane impls + ClusterMetrics) - import after PR3 lands."
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+      "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "ops"}},
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
+      "id": 6,
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
+          "expr": "sum(rate(stirling_cluster_ratelimit_rejected_total[5m]))",
+          "legendFormat": "rejections / sec",
+          "refId": "A"
+        }
+      ],
+      "title": "Rate-limit rejections (cluster-wide)",
+      "type": "timeseries",
+      "description": "Requires PR3 (Valkey backplane impls + ClusterMetrics) - import after PR3 lands."
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": ["stirling-pdf", "cluster"],
+  "templating": {
+    "list": [
+      {
+        "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {"from": "now-1h", "to": "now"},
+  "timepicker": {},
+  "timezone": "",
+  "title": "Stirling-PDF Cluster",
+  "uid": "stirling-cluster-phase1",
+  "version": 1,
+  "weekStart": ""
+}

docker/Dockerfile.unified

@@ -0,0 +1,95 @@
+# Stirling-PDF unified container. MODE env var selects role: FRONTEND, BACKEND, or BOTH.
+
+ARG BASE_VERSION=1.0.2
+ARG BASE_IMAGE=stirlingtools/stirling-pdf-base:${BASE_VERSION}
+
+# Stage 1: Build the Java application
+FROM gradle:9.3.1-jdk25 AS app-build
+
+ARG TASK_VERSION=3.49.1
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl ca-certificates \
+    && update-ca-certificates \
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && ARCH=$(dpkg --print-architecture) \
+    && curl -fsSL "https://github.com/go-task/task/releases/download/v${TASK_VERSION}/task_${TASK_VERSION}_linux_${ARCH}.deb" -o /tmp/task.deb \
+    && dpkg -i /tmp/task.deb \
+    && rm /tmp/task.deb \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+  --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+  --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+  --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+  --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"
+
+WORKDIR /app
+
+COPY build.gradle settings.gradle gradlew ./
+COPY gradle/                               gradle/
+COPY app/core/build.gradle                  app/core/
+COPY app/common/build.gradle                app/common/
+COPY app/proprietary/build.gradle           app/proprietary/
+
+COPY . .
+
+ARG STIRLING_FLAVOR=proprietary
+ENV STIRLING_FLAVOR=${STIRLING_FLAVOR}
+
+RUN STIRLING_FLAVOR=${STIRLING_FLAVOR} \
+    gradle clean build \
+      -PbuildWithFrontend=true \
+      -x spotlessApply -x spotlessCheck -x test -x sonarqube \
+      --no-daemon
+
+# Stage 2: Extract Spring Boot layers
+FROM eclipse-temurin:25-jre-noble AS jar-extract
+WORKDIR /tmp
+COPY --from=app-build /app/app/core/build/libs/*.jar app.jar
+RUN java -Djarmode=tools -jar app.jar extract --layers --destination /layers
+
+# Stage 3: Runtime
+FROM ${BASE_IMAGE}
+
+ARG VERSION_TAG
+
+WORKDIR /app
+
+COPY --link --from=jar-extract --chown=1000:1000 /layers/dependencies/           /app/
+COPY --link --from=jar-extract --chown=1000:1000 /layers/spring-boot-loader/     /app/
+COPY --link --from=jar-extract --chown=1000:1000 /layers/snapshot-dependencies/  /app/
+COPY --link --from=jar-extract --chown=1000:1000 /layers/application/            /app/
+
+COPY --link --chown=1000:1000 scripts/ /scripts/
+
+ENV MODE=BOTH \
+    BACKEND_INTERNAL_PORT=8081 \
+    STIRLING_AOT_ENABLE="false" \
+    STIRLING_JVM_PROFILE="balanced" \
+    JAVA_CUSTOM_OPTS="" \
+    HOME=/home/stirlingpdfuser \
+    PUID=1000 \
+    PGID=1000 \
+    UMASK=022 \
+    STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \
+    TMPDIR=/tmp/stirling-pdf
+
+RUN echo "${VERSION_TAG:-dev}" > /etc/stirling_version
+
+LABEL org.opencontainers.image.title="Stirling-PDF Unified" \
+      org.opencontainers.image.description="Unified container - selectable role via MODE env var." \
+      org.opencontainers.image.source="https://github.com/Stirling-Tools/Stirling-PDF" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.vendor="Stirling-Tools" \
+      org.opencontainers.image.version="${VERSION_TAG}"
+
+EXPOSE 8080/tcp
+STOPSIGNAL SIGTERM
+
+HEALTHCHECK --interval=30s --timeout=15s --start-period=120s --retries=5 \
+  CMD curl -fs --max-time 10 http://localhost:8080${SYSTEM_ROOTURIPATH:-''}/api/v1/info/status || exit 1
+
+# Map MODE to cluster node role at container start.
+ENTRYPOINT ["sh", "-c", "case \"$MODE\" in BACKEND) export CLUSTER_NODE_ROLE=worker;; FRONTEND) export CLUSTER_NODE_ROLE=web;; *) export CLUSTER_NODE_ROLE=both;; esac; exec tini -- /scripts/init.sh"]
+CMD []

docker/compose/docker-compose-cluster.yml

@@ -0,0 +1,194 @@
+# Reference clustered deployment: load balancer + N app + N engine + Valkey + Postgres.
+# Non-k8s enterprise self-host blueprint. /internal/* stays off the LB.
+#
+# Required env vars (set in a .env beside this file or via your secret store):
+#   CLUSTER_ENGINE_SHAREDSECRET  - generate: openssl rand -hex 32
+#   STIRLING_VALKEY_PASSWORD     - generate: openssl rand -hex 32
+#   POSTGRES_PASSWORD            - required
+#   STIRLING_PREMIUM_KEY         - SERVER/ENTERPRISE license key (cluster mode is a paid feature)
+
+services:
+  valkey:
+    image: valkey/valkey:8.0-alpine
+    container_name: stirling-valkey
+    restart: unless-stopped
+    command:
+      [
+        "valkey-server",
+        "--requirepass",
+        "${STIRLING_VALKEY_PASSWORD:?STIRLING_VALKEY_PASSWORD must be set in production}",
+        "--maxmemory",
+        "256mb",
+        "--maxmemory-policy",
+        "allkeys-lru"
+      ]
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "valkey-cli",
+          "-a",
+          "${STIRLING_VALKEY_PASSWORD:?required}",
+          "--no-auth-warning",
+          "ping"
+        ]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    # Do not publish 6379 in prod. Uncomment only for local valkey-cli introspection.
+    # ports:
+    #   - "6379:6379"
+    deploy:
+      resources:
+        limits:
+          memory: 384m
+        reservations:
+          memory: 128m
+    volumes:
+      - valkey-data:/data
+
+  postgres:
+    image: postgres:17-alpine
+    container_name: stirling-postgres
+    environment:
+      POSTGRES_USER: stirling
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set}
+      POSTGRES_DB: stirling
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+        reservations:
+          memory: 256m
+
+  app-1:
+    build:
+      context: ../..
+      dockerfile: docker/Dockerfile.unified
+    container_name: stirling-app-1
+    restart: unless-stopped
+    environment:
+      MODE: BOTH
+      CLUSTER_ENABLED: "true"
+      CLUSTER_BACKPLANE: valkey
+      CLUSTER_VALKEY_URL: redis://:${STIRLING_VALKEY_PASSWORD:?required}@valkey:6379
+      CLUSTER_NODE_ID: app-1
+      CLUSTER_NODE_INTERNALADDRESS: app-1:8080
+      CLUSTER_ENGINE_SHAREDSECRET: ${CLUSTER_ENGINE_SHAREDSECRET:?required}
+      AIENGINE_URL: http://engine-lb:5001
+      STIRLING_PREMIUM_KEY: ${STIRLING_PREMIUM_KEY:?STIRLING_PREMIUM_KEY required for cluster mode}
+      SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/stirling
+      SPRING_DATASOURCE_USERNAME: stirling
+      SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?required}
+    depends_on:
+      valkey:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      engine-lb:
+        condition: service_started
+    deploy:
+      resources:
+        limits:
+          memory: 2g
+        reservations:
+          memory: 512m
+
+  app-2:
+    build:
+      context: ../..
+      dockerfile: docker/Dockerfile.unified
+    container_name: stirling-app-2
+    restart: unless-stopped
+    environment:
+      MODE: BOTH
+      CLUSTER_ENABLED: "true"
+      CLUSTER_BACKPLANE: valkey
+      CLUSTER_VALKEY_URL: redis://:${STIRLING_VALKEY_PASSWORD:?required}@valkey:6379
+      CLUSTER_NODE_ID: app-2
+      CLUSTER_NODE_INTERNALADDRESS: app-2:8080
+      CLUSTER_ENGINE_SHAREDSECRET: ${CLUSTER_ENGINE_SHAREDSECRET:?required}
+      AIENGINE_URL: http://engine-lb:5001
+      STIRLING_PREMIUM_KEY: ${STIRLING_PREMIUM_KEY:?STIRLING_PREMIUM_KEY required for cluster mode}
+      SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/stirling
+      SPRING_DATASOURCE_USERNAME: stirling
+      SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?required}
+    depends_on:
+      valkey:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      engine-lb:
+        condition: service_started
+    deploy:
+      resources:
+        limits:
+          memory: 2g
+        reservations:
+          memory: 512m
+
+  engine-1:
+    build:
+      context: ../../engine
+      dockerfile: Dockerfile.dev
+    container_name: stirling-engine-1
+    env_file: ../../engine/.env
+    environment:
+      STIRLING_ENGINE_SHARED_SECRET: ${CLUSTER_ENGINE_SHAREDSECRET:?required}
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+
+  engine-2:
+    build:
+      context: ../../engine
+      dockerfile: Dockerfile.dev
+    container_name: stirling-engine-2
+    env_file: ../../engine/.env
+    environment:
+      STIRLING_ENGINE_SHARED_SECRET: ${CLUSTER_ENGINE_SHAREDSECRET:?required}
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+
+  # Internal round-robin LB for the engine tier - not exposed to the host.
+  engine-lb:
+    image: nginx:1.27-alpine
+    container_name: stirling-engine-lb
+    volumes:
+      - ./engine-nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - engine-1
+      - engine-2
+    deploy:
+      resources:
+        limits:
+          memory: 64m
+
+  lb:
+    image: nginx:1.27-alpine
+    container_name: stirling-lb
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - app-1
+      - app-2
+    deploy:
+      resources:
+        limits:
+          memory: 128m
+
+volumes:
+  valkey-data:
+  postgres-data:

docker/compose/engine-nginx.conf

@@ -0,0 +1,27 @@
+# nginx LB for the AI engine tier of docker-compose-cluster.yml.
+# Round-robin is safe (engine is stateless). The separate LB hop exists because the JVM
+# caches DNS lookups for the process lifetime, so compose round-robin DNS would pin each
+# JVM to one engine and defeat scaling.
+
+events { worker_connections 1024; }
+
+http {
+    upstream stirling_engine {
+        server engine-1:5001;
+        server engine-2:5001;
+    }
+
+    server {
+        listen 5001;
+
+        # Matches JVM multipart cap; nginx default 1MB would 413.
+        client_max_body_size 2000m;
+
+        location / {
+            proxy_pass http://stirling_engine;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $host;
+            proxy_read_timeout 600s;
+        }
+    }
+}

docker/compose/nginx.conf

@@ -0,0 +1,33 @@
+# nginx LB for docker-compose-cluster.yml.
+# Sends all traffic to the Java app (including /api/v1/ai/*, which the app proxies to
+# the engine with X-Engine-Auth attestation). /internal/* is 404'd. The engine tier
+# is never exposed directly to clients.
+
+events { worker_connections 1024; }
+
+http {
+    upstream stirling_app {
+        # Sticky sessions required - see deploy/aws/README.md for ip_hash caveat + alternatives.
+        ip_hash;
+        server app-1:8080;
+        server app-2:8080;
+    }
+
+    server {
+        listen 8080;
+
+        # Matches JVM multipart.max-file-size; nginx default 1MB would 413.
+        client_max_body_size 2000m;
+
+        location /internal/ {
+            return 404;
+        }
+
+        location / {
+            proxy_pass http://stirling_app;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $host;
+            proxy_read_timeout 600s;
+        }
+    }
+}

engine/Dockerfile.dev

@@ -4,6 +4,7 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
 WORKDIR /app
 
 COPY pyproject.toml uv.lock ./
+COPY src/ ./src/
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen