Add pipelines page to portal (#6818)

# Description of Changes

Connect pipelines page to the backend. Note that this is really half an
implementation because the portal doesn't have access to the tools list
and their settings, but I can't fix that without re-architecture work,
which I'll do in another PR, then come back to finish this off in a new
PR.

<img width="786" height="579" alt="image"
src="https://github.com/user-attachments/assets/d3f06110-a35d-4d48-a2f9-1edb900c5c35"
/>

<img width="1232" height="519" alt="image"
src="https://github.com/user-attachments/assets/9f344648-ea45-498d-9e84-9558a3999838"
/>

app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java

@@ -2,6 +2,7 @@ package stirling.software.proprietary.policy.controller;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -52,11 +53,15 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
 import stirling.software.proprietary.policy.model.PolicyRun;
 import stirling.software.proprietary.policy.model.PolicyRunStatus;
 import stirling.software.proprietary.policy.model.PolicyRunView;
+import stirling.software.proprietary.policy.overview.PoliciesOverviewResponse;
+import stirling.software.proprietary.policy.overview.PolicyOverviewService;
 import stirling.software.proprietary.policy.progress.PolicyProgressListener;
 import stirling.software.proprietary.policy.source.SourceAccessGuard;
 import stirling.software.proprietary.policy.source.SourceStore;
 import stirling.software.proprietary.policy.store.PolicyStore;
+import stirling.software.proprietary.policy.trigger.PolicyTrigger;
 import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
+import stirling.software.proprietary.policy.trigger.TriggerInfo;
 
 /**
  * Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
@@ -80,6 +85,8 @@ public class PolicyController {
     private final PolicyAccessGuard policyAccessGuard;
     private final PolicyManagementAuthority policyManagementAuthority;
     private final PolicyTriggerManager policyTriggerManager;
+    private final PolicyOverviewService policyOverviewService;
+    private final List<PolicyTrigger> policyTriggers;
     private final ApplicationProperties applicationProperties;
     private final TempFileManager tempFileManager;
     private final JobOwnershipService jobOwnershipService;
@@ -287,6 +294,31 @@ public class PolicyController {
         return policyAccessGuard.visibleFrom(policyStore);
     }
 
+    @GetMapping("/overview")
+    @Operation(
+            summary = "Pipelines overview",
+            description =
+                    "Returns the KPI strip plus one row per policy the caller's team owns, each with"
+                            + " its referenced sources resolved to names, its pipeline steps, and a"
+                            + " trigger/output summary. Backs the portal's all-pipelines surface.")
+    public PoliciesOverviewResponse overview() {
+        return policyOverviewService.overview();
+    }
+
+    @GetMapping("/triggers")
+    @Operation(
+            summary = "List available triggers",
+            description =
+                    "Lists each trigger kind with whether it needs a source and which source types"
+                            + " it supports, so the UI can offer triggers and pair them with the"
+                            + " right sources.")
+    public List<TriggerInfo> triggers() {
+        return policyTriggers.stream()
+                .map(TriggerInfo::of)
+                .sorted(Comparator.comparing(TriggerInfo::type))
+                .toList();
+    }
+
     @GetMapping("/{policyId}")
     @Operation(summary = "Get a policy by id")
     public ResponseEntity<Policy> getPolicy(@PathVariable String policyId) {
@@ -337,6 +369,26 @@ public class PolicyController {
         return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
     }
 
+    @PostMapping("/{policyId}/trigger")
+    @Operation(
+            summary = "Run a stored policy against its sources",
+            description =
+                    "Pulls the policy's configured sources and runs the pipeline now, regardless of"
+                            + " the enabled flag (which only gates automatic triggering). Returns"
+                            + " the ids of the runs started; poll the run-status endpoint for each."
+                            + " Empty when the sources yielded no work to do.")
+    public ResponseEntity<List<String>> trigger(@PathVariable String policyId) {
+        Policy policy =
+                policyStore
+                        .get(policyId)
+                        .filter(policyAccessGuard::canAccess)
+                        .orElseThrow(
+                                () ->
+                                        new ResponseStatusException(
+                                                HttpStatus.NOT_FOUND, "No policy: " + policyId));
+        return ResponseEntity.accepted().body(policyRunner.run(policy));
+    }
+
     private static void requireRunnable(PipelineDefinition definition) {
         if (definition.steps().isEmpty()) {
             throw new ResponseStatusException(

app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java

@@ -1,6 +1,7 @@
 package stirling.software.proprietary.policy.engine;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 
@@ -41,14 +42,15 @@ public class PolicyRunner {
      * Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
      * one failure does not affect the others. No sources means one run with no input (generator
      * pipeline). Missing or disabled sources are skipped so one broken reference does not stop the
-     * rest.
+     * rest. Returns the ids of the runs it started (empty when sources yielded no work), so a
+     * manual trigger can report back which runs to follow.
      */
-    public void run(Policy policy) {
+    public List<String> run(Policy policy) {
         List<String> sourceIds = policy.sourceIds();
         if (sourceIds.isEmpty()) {
-            startRun(policy, PolicyInputs.of(List.of()), unused -> {});
-            return;
+            return List.of(startRun(policy, PolicyInputs.of(List.of()), unused -> {}));
         }
+        List<String> runIds = new ArrayList<>();
         for (String sourceId : sourceIds) {
             Source source = sourceStore.get(sourceId).orElse(null);
             if (source == null) {
@@ -63,8 +65,9 @@ public class PolicyRunner {
                         policy.id());
                 continue;
             }
-            pullAndRun(policy, source.toInputSpec());
+            runIds.addAll(pullAndRun(policy, source.toInputSpec()));
         }
+        return runIds;
     }
 
     /** Run a stored policy on caller-supplied files (e.g. manual upload), bypassing its sources. */
@@ -79,14 +82,14 @@ public class PolicyRunner {
         return policyEngine.submit(definition, inputs, listener);
     }
 
-    private void pullAndRun(Policy policy, InputSpec spec) {
+    private List<String> pullAndRun(Policy policy, InputSpec spec) {
         InputSource source = sourceFor(spec);
         if (source == null) {
             log.warn(
                     "No input source for type '{}' (policy {}); skipping",
                     spec.type(),
                     policy.id());
-            return;
+            return List.of();
         }
         List<ResolvedInput> work;
         try {
@@ -97,19 +100,22 @@ public class PolicyRunner {
                     spec.type(),
                     policy.id(),
                     e.getMessage());
-            return;
+            return List.of();
         }
+        List<String> runIds = new ArrayList<>();
         for (ResolvedInput unit : work) {
-            startRun(policy, unit.inputs(), unit.onComplete());
+            runIds.add(startRun(policy, unit.inputs(), unit.onComplete()));
         }
+        return runIds;
     }
 
-    private void startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
+    private String startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
         log.info("Running policy {} ({})", policy.id(), policy.name());
         PolicyRunHandle handle =
                 policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP);
         handle.completion()
                 .whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
+        return handle.runId();
     }
 
     private static boolean succeeded(PolicyRun run, Throwable throwable) {

app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PoliciesOverviewResponse.java

@@ -0,0 +1,6 @@
+package stirling.software.proprietary.policy.overview;
+
+import java.util.List;
+
+/** The Pipelines overview payload: a KPI strip plus one row per policy. */
+public record PoliciesOverviewResponse(List<PolicyKpi> kpis, List<PolicyView> pipelines) {}

app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PolicyKpi.java

@@ -0,0 +1,4 @@
+package stirling.software.proprietary.policy.overview;
+
+/** One headline figure in the Pipelines overview strip. */
+public record PolicyKpi(long value, String description) {}

app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PolicyOverviewService.java

@@ -0,0 +1,102 @@
+package stirling.software.proprietary.policy.overview;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
+import org.springframework.stereotype.Service;
+
+import lombok.RequiredArgsConstructor;
+
+import stirling.software.proprietary.policy.config.PolicyAccessGuard;
+import stirling.software.proprietary.policy.model.OutputSpec;
+import stirling.software.proprietary.policy.model.PipelineStep;
+import stirling.software.proprietary.policy.model.Policy;
+import stirling.software.proprietary.policy.model.TriggerConfig;
+import stirling.software.proprietary.policy.source.Source;
+import stirling.software.proprietary.policy.source.SourceAccessGuard;
+import stirling.software.proprietary.policy.source.SourceStore;
+import stirling.software.proprietary.policy.store.PolicyStore;
+
+/**
+ * Builds the Pipelines overview: every policy the caller's team owns, each annotated with its
+ * referenced sources (resolved to display names), its pipeline steps, and a trigger/output summary.
+ * Source names are resolved from the team's sources in memory rather than persisted on the policy,
+ * so the view always reflects the live source set. This is the "all pipelines" admin surface; the
+ * user-facing Policies page builds only a friendly subset of the same backend policies.
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnBooleanProperty(name = "policies.enabled")
+public class PolicyOverviewService {
+
+    private final PolicyStore policyStore;
+    private final SourceStore sourceStore;
+    private final PolicyAccessGuard policyAccessGuard;
+    private final SourceAccessGuard sourceAccessGuard;
+
+    public PoliciesOverviewResponse overview() {
+        List<Policy> policies = policyAccessGuard.visibleFrom(policyStore);
+        Map<String, String> sourceNames = sourceNames();
+
+        List<PolicyView> views =
+                policies.stream()
+                        .map(policy -> toView(policy, sourceNames))
+                        .sorted(
+                                Comparator.comparing(
+                                        PolicyView::name, String.CASE_INSENSITIVE_ORDER))
+                        .toList();
+
+        return new PoliciesOverviewResponse(buildKpis(policies), views);
+    }
+
+    /** Display names for every source the caller's team can see, keyed by source id. */
+    private Map<String, String> sourceNames() {
+        Map<String, String> names = new HashMap<>();
+        for (Source source : sourceAccessGuard.visibleFrom(sourceStore)) {
+            names.put(source.id(), source.name());
+        }
+        return names;
+    }
+
+    private static PolicyView toView(Policy policy, Map<String, String> sourceNames) {
+        List<PolicyView.SourceRef> sources =
+                policy.sourceIds().stream()
+                        // An unresolved id (source deleted, or not visible) falls back to the id so
+                        // the row still renders rather than dropping the reference silently.
+                        .map(id -> new PolicyView.SourceRef(id, sourceNames.getOrDefault(id, id)))
+                        .toList();
+        List<String> steps = policy.steps().stream().map(PipelineStep::operation).toList();
+        return new PolicyView(
+                policy.id(),
+                policy.name(),
+                policy.enabled(),
+                policy.enabled() ? "active" : "paused",
+                triggerSummary(policy.trigger()),
+                sources,
+                steps,
+                outputSummary(policy.output()),
+                policy.owner());
+    }
+
+    /** A null trigger is a manual-only policy; otherwise the trigger's type keys the summary. */
+    private static String triggerSummary(TriggerConfig trigger) {
+        return trigger == null ? "manual" : trigger.type();
+    }
+
+    private static String outputSummary(OutputSpec output) {
+        return output == null ? "inline" : output.type();
+    }
+
+    private static List<PolicyKpi> buildKpis(List<Policy> policies) {
+        long total = policies.size();
+        long active = policies.stream().filter(Policy::enabled).count();
+        long paused = total - active;
+        return List.of(
+                new PolicyKpi(total, "pipelines"),
+                new PolicyKpi(active, "running automatically"),
+                new PolicyKpi(paused, "paused"));
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PolicyView.java

@@ -0,0 +1,24 @@
+package stirling.software.proprietary.policy.overview;
+
+import java.util.List;
+
+/**
+ * One row in the Pipelines overview: a stored policy shown for the admin portal, with its
+ * referenced sources resolved to names and its pipeline summarised. The portal's "all pipelines"
+ * surface lists every backend policy (the user-facing Policies page builds only a friendly subset
+ * of these).
+ */
+public record PolicyView(
+        String id,
+        String name,
+        boolean enabled,
+        String status,
+        String trigger,
+        List<SourceRef> sources,
+        List<String> steps,
+        String output,
+        String owner) {
+
+    /** A source a policy pulls documents from, resolved to its display name. */
+    public record SourceRef(String id, String name) {}
+}

app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java

@@ -27,6 +27,7 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.policy.config.FolderAccessGuard;
 import stirling.software.proprietary.policy.engine.PolicyRunner;
 import stirling.software.proprietary.policy.input.InputSource;
 import stirling.software.proprietary.policy.model.InputSpec;
@@ -74,6 +75,16 @@ public class FolderWatchTrigger implements PolicyTrigger {
         return TYPE;
     }
 
+    @Override
+    public boolean requiresSource() {
+        return true;
+    }
+
+    @Override
+    public Set<String> supportedSourceTypes() {
+        return Set.of(FolderAccessGuard.FOLDER_TYPE);
+    }
+
     @Override
     public void validate(Policy policy) {
         if (watchDirsOf(policy).isEmpty()) {

app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java

@@ -1,5 +1,7 @@
 package stirling.software.proprietary.policy.trigger;
 
+import java.util.Set;
+
 import stirling.software.proprietary.policy.model.Policy;
 
 /**
@@ -11,6 +13,24 @@ public interface PolicyTrigger {
     /** Matches {@code TriggerConfig.type()}. */
     String type();
 
+    /**
+     * Whether this trigger needs at least one compatible input source to function. A schedule fires
+     * on the clock regardless of sources, so it is false; folder-watch derives the directories it
+     * watches from the policy's sources, so it is true. Drives whether the UI offers the trigger.
+     */
+    default boolean requiresSource() {
+        return false;
+    }
+
+    /**
+     * The source {@code type()}s this trigger is compatible with (e.g. {@code "folder"}). Empty
+     * means source-agnostic (no constraint). Lets the UI offer a trigger only when a compatible
+     * source is selected, without hard-coding the relationship.
+     */
+    default Set<String> supportedSourceTypes() {
+        return Set.of();
+    }
+
     /**
      * Validate at save time so misconfiguration fails fast, not at fire time. Receives the whole
      * {@link Policy} so triggers that depend on the policy's sources (folder-watch) can check that.

app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/TriggerInfo.java

@@ -0,0 +1,18 @@
+package stirling.software.proprietary.policy.trigger;
+
+import java.util.List;
+
+/**
+ * Describes an available trigger for the admin UI: its {@code type} (matching {@code
+ * TriggerConfig.type()}), whether it needs a compatible source, and which source types it works
+ * with. Lets the UI list supported triggers and pair them with sources without hard-coding the set.
+ */
+public record TriggerInfo(String type, boolean requiresSource, List<String> supportedSourceTypes) {
+
+    public static TriggerInfo of(PolicyTrigger trigger) {
+        return new TriggerInfo(
+                trigger.type(),
+                trigger.requiresSource(),
+                List.copyOf(trigger.supportedSourceTypes()));
+    }
+}