From 84d4455682a3a89fbde79b612404a106ccdc2a69 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Fri, 10 Jul 2026 14:27:40 +0100 Subject: [PATCH] Add virtual Editor source (#6959) # Description of Changes Adds Editor source permanently available in the Sources list. Excludes it from the Pipelines list of available sources currently because it's not a real source on the backend, so attempting to connect to it causes an error. It'd be nice to extend in the future to be able to set up policies in the editor from the pipelines page, but this'll do for now. --- .../policy/controller/PolicyController.java | 22 +++++- .../policy/engine/PolicyRunner.java | 11 ++- .../policy/source/EditorSource.java | 27 +++++++ .../policy/source/SourceAccessGuard.java | 7 ++ .../policy/source/SourceController.java | 18 +++++ .../policy/source/SourceOverviewService.java | 53 +++++++++++++- .../controller/PolicyControllerTest.java | 16 ++++ .../policy/engine/PolicyRunnerTest.java | 32 +++++++- .../policy/source/SourceControllerTest.java | 27 +++++++ .../source/SourceOverviewServiceTest.java | 73 +++++++++++++++++-- .../public/locales/en-US/translation.toml | 2 + .../components/sources/SourceDetailCard.tsx | 65 ++++++++++------- .../components/sources/SourceDetailPanel.tsx | 14 +++- .../components/sources/SourcesTable.tsx | 18 +++-- .../portal/components/sources/sourceTypes.ts | 7 ++ .../src/portal/mocks/handlers/sources.ts | 36 ++++++++- .../src/portal/views/PipelineBuilder.tsx | 8 +- .../editor/src/portal/views/Sources.test.tsx | 33 +++++++++ 18 files changed, 418 insertions(+), 51 deletions(-) create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 045698c015..3c9b56128a 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -60,7 +60,9 @@ import stirling.software.proprietary.policy.model.PolicyRunView; import stirling.software.proprietary.policy.overview.PoliciesOverviewResponse; import stirling.software.proprietary.policy.overview.PolicyOverviewService; import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.policy.source.EditorSource; import stirling.software.proprietary.policy.source.SourceAccessGuard; +import stirling.software.proprietary.policy.source.SourceDocCounter; import stirling.software.proprietary.policy.source.SourceStore; import stirling.software.proprietary.policy.store.PolicyStore; import stirling.software.proprietary.policy.trigger.PolicyTrigger; @@ -86,6 +88,7 @@ public class PolicyController { private final PolicyStore policyStore; private final SourceStore sourceStore; private final SourceAccessGuard sourceAccessGuard; + private final SourceDocCounter docCounter; private final PolicyValidator policyValidator; private final PolicyAccessGuard policyAccessGuard; private final PolicyManagementAuthority policyManagementAuthority; @@ -112,9 +115,10 @@ public class PolicyController { throws IOException { requireRunnable(definition); PolicyInputs inputs = toInputs(files); - String runId = - policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId(); - return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); + PolicyRunHandle handle = + policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP); + recordEditorDocs(inputs); + return ResponseEntity.accepted().body(new JobResponse<>(true, handle.runId(), null)); } @PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @@ -136,6 +140,7 @@ public class PolicyController { emitter.onError(e -> log.warn("Policy run SSE emitter error", e)); PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter)); + recordEditorDocs(inputs); // whenComplete runs on the worker thread after the run finishes, so the terminal event // never races the step events. handle.completion() @@ -492,6 +497,17 @@ public class PolicyController { } } + /** + * Ad-hoc runs (AI / one-off pipelines) are still editor activity, so their supplied documents + * feed the same virtual editor source as stored editor policies, counted against the caller's + * team. A run with no primary documents (generator pipeline) records nothing. + */ + private void recordEditorDocs(PolicyInputs inputs) { + docCounter.record( + EditorSource.counterKey(sourceAccessGuard.currentTeamId()), + inputs.primary().size()); + } + /** * Turn the typed run files into engine {@link PolicyInputs}: the primary documents plus the * named supporting-file store, where each asset's {@code key} is the name a step references diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java index 7731d936bd..8575e95e61 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java @@ -21,6 +21,7 @@ import stirling.software.proprietary.policy.model.PolicyInputs; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.PolicyRunStatus; import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.policy.source.EditorSource; import stirling.software.proprietary.policy.source.Source; import stirling.software.proprietary.policy.source.SourceDocCounter; import stirling.software.proprietary.policy.source.SourceStore; @@ -98,10 +99,16 @@ public class PolicyRunner { return context.outcome(runIds); } - /** Run a stored policy on caller-supplied files (e.g. manual upload), bypassing its sources. */ + /** + * Run a stored policy on caller-supplied files (e.g. an editor upload), bypassing its sources. + * The supplied documents are still counted against the virtual {@link EditorSource}, scoped to + * the policy's team, so the Sources overview reports the whole team's editor throughput. + */ public PolicyRunHandle runWith( Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { - return policyEngine.runPolicy(policy, inputs, listener); + PolicyRunHandle handle = policyEngine.runPolicy(policy, inputs, listener); + docCounter.record(EditorSource.counterKey(policy.teamId()), inputs.primary().size()); + return handle; } /** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */ diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java new file mode 100644 index 0000000000..3a7a73df5b --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/EditorSource.java @@ -0,0 +1,27 @@ +package stirling.software.proprietary.policy.source; + +/** + * The Editor as a virtual, always-present source. Unlike a persisted {@link Source} it is neither + * stored nor configurable: it stands for the documents a team processes by running policies from + * the in-app editor (the {@code POST /api/v1/policies/{id}/run} path). Its throughput is tracked + * through {@link SourceDocCounter} under a synthetic, team-scoped key, so each team sees only its + * own editor activity and the client is only ever handed the opaque {@link #ID}, never a team. + */ +public final class EditorSource { + + /** The single, stable id and type the client sees for the editor row. */ + public static final String ID = "editor"; + + public static final String TYPE = "editor"; + + private EditorSource() {} + + /** + * The per-team {@link SourceDocCounter} key. A {@code null} team (login disabled / self-hosted + * single user) shares one global bucket; otherwise counts are partitioned by team so a team's + * total aggregates every member's editor runs and no other team's. + */ + public static String counterKey(Long teamId) { + return teamId == null ? ID : ID + ":" + teamId; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java index 1db5663fd2..f5a14282c9 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceAccessGuard.java @@ -34,6 +34,13 @@ public class SourceAccessGuard { /** Team a new source is stamped with: the creator's team. {@code null} when login disabled. */ public Long teamForNewSource() { + return currentTeamId(); + } + + /** + * The current user's team (what scopes their sources), or {@code null} when login is disabled. + */ + public Long currentTeamId() { return enforced() ? policyManagementAuthority.currentUserTeamId() : null; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java index 508e33a335..524b8c369c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceController.java @@ -88,6 +88,10 @@ public class SourceController { "The trailing 30-day per-day document series (oldest first) for the source's" + " sparkline.") public ResponseEntity> documentCounts(@PathVariable String sourceId) { + // The editor is virtual: its series is tracked per team, not against a persisted source. + if (EditorSource.ID.equals(sourceId)) { + return ResponseEntity.ok(overviewService.editorDailySeries()); + } return sourceStore .get(sourceId) .filter(sourceAccessGuard::canAccess) @@ -104,6 +108,7 @@ public class SourceController { + " matching source type.") public ResponseEntity save(@RequestBody Source source) { requireSourceEditingAllowed(); + requireNotEditor(source.id(), source.type()); Source owned = withStoredSecrets(resolveOwnership(source)); try { validateConfig(owned); @@ -125,6 +130,7 @@ public class SourceController { + " so the connection can't be pulled out from under a live policy.") public ResponseEntity delete(@PathVariable String sourceId) { requireSourceEditingAllowed(); + requireNotEditor(sourceId, null); Source source = sourceStore.get(sourceId).filter(sourceAccessGuard::canAccess).orElse(null); if (source == null) { return ResponseEntity.notFound().build(); @@ -238,6 +244,18 @@ public class SourceController { } } + /** + * The editor is a built-in, virtual source: it is always present and cannot be created, edited, + * or deleted like a persisted connection. Reject any attempt to touch it by id or type. + */ + private static void requireNotEditor(String id, String type) { + if (EditorSource.ID.equals(id) || EditorSource.TYPE.equals(type)) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, + "The editor is a built-in source and cannot be created, edited, or deleted"); + } + } + /** Names of the caller's visible policies that reference the given source. */ private List referencingPolicyNames(String sourceId) { return policyAccessGuard.visibleFrom(policyStore).stream() diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java index af7ec3b11b..f202a30e23 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/source/SourceOverviewService.java @@ -41,7 +41,7 @@ public class SourceOverviewService { Map docStats = docCounter.statsFor(sources.stream().map(Source::id).toList()); - List views = + List persisted = sources.stream() .map( source -> @@ -56,7 +56,13 @@ public class SourceOverviewService { .thenComparing(SourceView::name)) .toList(); - return new SourcesResponse(buildKpis(views), views); + // The editor is a built-in source: always present and pinned first. The KPI strip counts + // only the connections a team configures, so the editor is left out of the KPIs. + List views = new ArrayList<>(); + views.add(editorView(policies)); + views.addAll(persisted); + + return new SourcesResponse(buildKpis(persisted), views); } /** @@ -67,6 +73,49 @@ public class SourceOverviewService { return docCounter.dailySeriesFor(sourceId); } + /** The 30-day daily editor document series (oldest first) for the caller's team. */ + public List editorDailySeries() { + return docCounter.dailySeriesFor( + EditorSource.counterKey(sourceAccessGuard.currentTeamId())); + } + + /** + * The always-present editor row. It has no stored config; its documents are those the team has + * processed by running policies from the editor, and it is "used by" every policy that targets + * the editor as its source. + */ + private SourceView editorView(List policies) { + String key = EditorSource.counterKey(sourceAccessGuard.currentTeamId()); + DocStats docs = docCounter.statsFor(List.of(key)).getOrDefault(key, DocStats.ZERO); + List refs = + policies.stream() + .filter(SourceOverviewService::runsFromEditor) + .map(policy -> new SourceView.PolicyRef(policy.id(), policy.name())) + .toList(); + return new SourceView( + EditorSource.ID, + "Editor", + EditorSource.TYPE, + "active", + refs.size(), + refs, + List.of(), + docs.total(), + docs.last24h(), + docs.last30d()); + } + + /** + * Whether a policy runs from the editor. Editor membership is carried in the policy's output + * metadata ({@code output.options.sources}) - a client-side list the editor writes when a + * policy targets it - rather than as a persisted {@code sourceId}, because the editor is + * virtual and has no stored source to reference. + */ + private static boolean runsFromEditor(Policy policy) { + Object sources = policy.output().options().get("sources"); + return sources instanceof List list && list.contains(EditorSource.ID); + } + /** Policies referencing each source id, across the caller's visible policies. */ private static Map> referencesBySource(List policies) { Map> bySource = new HashMap<>(); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java index 5d8961a46e..c02945fdad 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java @@ -45,7 +45,9 @@ import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.PolicyRunView; import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.policy.source.EditorSource; import stirling.software.proprietary.policy.source.SourceAccessGuard; +import stirling.software.proprietary.policy.source.SourceDocCounter; import stirling.software.proprietary.policy.source.SourceStore; import stirling.software.proprietary.policy.trigger.PolicyTriggerManager; import stirling.software.proprietary.util.SecretMasker; @@ -59,6 +61,7 @@ class PolicyControllerTest { @Mock private stirling.software.proprietary.policy.store.PolicyStore policyStore; @Mock private SourceStore sourceStore; @Mock private SourceAccessGuard sourceAccessGuard; + @Mock private SourceDocCounter docCounter; @Mock private PolicyValidator policyValidator; @Mock private PolicyAccessGuard policyAccessGuard; @Mock private PolicyManagementAuthority policyManagementAuthority; @@ -92,6 +95,7 @@ class PolicyControllerTest { policyStore, sourceStore, sourceAccessGuard, + docCounter, policyValidator, policyAccessGuard, policyManagementAuthority, @@ -166,6 +170,18 @@ class PolicyControllerTest { assertThat(response.getBody().getJobId()).isEqualTo("run-1"); } + @Test + @DisplayName("feeds the editor source, scoped to the caller's team") + void adHocRunFeedsTheEditorSource() throws Exception { + when(policyRunner.runAdHoc(any(), any(), eq(PolicyProgressListener.NOOP))) + .thenReturn(handle("run-1")); + when(sourceAccessGuard.currentTeamId()).thenReturn(3L); + + controller.run(definitionWithStep(), new PolicyRunFiles()); + + verify(docCounter).record(EditorSource.counterKey(3L), 0L); + } + @Test @DisplayName("rejects a pipeline with no steps") void rejectsEmptyPipeline() { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java index ef4c0300da..e21f371305 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.io.ByteArrayResource; import stirling.software.proprietary.policy.input.InputSource; import stirling.software.proprietary.policy.input.ResolveContext; @@ -41,6 +42,7 @@ import stirling.software.proprietary.policy.model.PolicyInputs; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.PolicyRunStatus; import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.policy.source.EditorSource; import stirling.software.proprietary.policy.source.InProcessSourceDocCounter; import stirling.software.proprietary.policy.source.InProcessSourceStore; import stirling.software.proprietary.policy.source.Source; @@ -58,6 +60,7 @@ class PolicyRunnerTest { @Mock private ProcessedLedger processedLedger; private final SourceStore sourceStore = new InProcessSourceStore(); + private final InProcessSourceDocCounter docCounter = new InProcessSourceDocCounter(); private PolicyRunner runner; @BeforeEach @@ -67,7 +70,7 @@ class PolicyRunnerTest { policyEngine, List.of(folderSource), sourceStore, - new InProcessSourceDocCounter(), + docCounter, processedLedger); } @@ -295,6 +298,33 @@ class PolicyRunnerTest { verifyNoInteractions(folderSource); } + @Test + void runWithRecordsSuppliedDocsAgainstTheEditorSourceForThePolicyTeam() { + Policy policy = + new Policy( + "p1", + "p", + "owner", + true, + null, + List.of(), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline(), + 7L); + PolicyInputs inputs = + PolicyInputs.of( + List.of( + new ByteArrayResource("a".getBytes()), + new ByteArrayResource("b".getBytes()))); + when(policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP)) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.runWith(policy, inputs, PolicyProgressListener.NOOP); + + String key = EditorSource.counterKey(7L); + assertEquals(2, docCounter.statsFor(List.of(key)).get(key).total()); + } + /** Persists each spec as a source and returns a policy referencing them by id. */ private Policy policy(List sources) { List sourceIds = diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java index e71deaf00a..295b4785b8 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceControllerTest.java @@ -110,6 +110,33 @@ class SourceControllerTest { assertEquals(404, controller.delete("nope").getStatusCode().value()); } + @Test + void documentCountsForTheEditorReturnsTheTeamSeries() { + ResponseEntity> response = controller.documentCounts(EditorSource.ID); + + assertEquals(200, response.getStatusCode().value()); + assertEquals(30, response.getBody().size()); + } + + @Test + void theEditorIsBuiltInAndCannotBeDeleted() { + ResponseStatusException ex = + assertThrows( + ResponseStatusException.class, () -> controller.delete(EditorSource.ID)); + + assertEquals(400, ex.getStatusCode().value()); + } + + @Test + void theEditorIsBuiltInAndCannotBeSaved() { + Source editor = new Source(null, "Editor", "editor", Map.of(), true, null, null); + + ResponseStatusException ex = + assertThrows(ResponseStatusException.class, () -> controller.save(editor)); + + assertEquals(400, ex.getStatusCode().value()); + } + @Test void readsReturnSecretsAsTheRedactionSentinel() { Source saved = sourceStore.save(s3Source("shh")); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java index 275f076db3..3f63e18407 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/source/SourceOverviewServiceTest.java @@ -56,9 +56,10 @@ class SourceOverviewServiceTest { SourcesResponse response = service.overview(); - assertEquals(3, response.sources().size()); - // Sorted most-referenced first, so the shared source A leads. - assertEquals(a.id(), response.sources().get(0).id()); + assertEquals(4, response.sources().size()); + // The built-in editor is pinned first; persisted sources follow, most-referenced leading. + assertEquals(EditorSource.ID, response.sources().get(0).id()); + assertEquals(a.id(), response.sources().get(1).id()); SourceView av = find(response, a.id()); assertEquals(2, av.referenceCount()); @@ -123,13 +124,57 @@ class SourceOverviewServiceTest { SourcesResponse response = scoped.overview(); - assertEquals(1, response.sources().size()); - SourceView view = response.sources().get(0); - assertEquals(ours.id(), view.id()); + assertEquals(2, response.sources().size()); + assertEquals(EditorSource.ID, response.sources().get(0).id()); + SourceView view = find(response, ours.id()); assertEquals(1, view.referenceCount()); assertEquals(List.of(1L, 1L, 0L), response.kpis().stream().map(SourceKpi::value).toList()); } + @Test + void theEditorSourceIsAlwaysPresentEvenWithNoConnections() { + SourcesResponse response = service.overview(); + + assertEquals(1, response.sources().size()); + SourceView editor = response.sources().get(0); + assertEquals(EditorSource.ID, editor.id()); + assertEquals("editor", editor.type()); + assertEquals("active", editor.status()); + assertEquals(0, editor.referenceCount()); + // KPIs describe configured connections, so the built-in editor is left out of them. + assertEquals(List.of(0L, 0L, 0L), response.kpis().stream().map(SourceKpi::value).toList()); + } + + @Test + void theEditorSourceIsUsedByEveryPolicyThatRunsFromIt() { + editorPolicy("Redact on upload"); + editorPolicy("Classify on upload"); + // A folder-sourced policy does not target the editor, so it must not inflate the count. + policyReferencing("Folder sweep", source("Folder", "/f").id()); + + SourceView editor = find(service.overview(), EditorSource.ID); + + assertEquals(2, editor.referenceCount()); + assertTrue( + editor.referencingPolicies().stream() + .map(SourceView.PolicyRef::name) + .toList() + .containsAll(List.of("Redact on upload", "Classify on upload"))); + } + + @Test + void theEditorSourceReportsTheTeamsRecordedDocumentThroughput() { + // Login disabled, so the team is null and the editor shares the global counter bucket. + docCounter.record(EditorSource.counterKey(null), 4); + docCounter.record(EditorSource.counterKey(null), 6); + + SourceView editor = find(service.overview(), EditorSource.ID); + + assertEquals(10, editor.docsTotal()); + assertEquals(10, editor.docs24h()); + assertEquals(10, editor.docs30d()); + } + @Test void documentCountsReflectRecordedDocs() { Source a = source("A", "/a"); @@ -177,6 +222,22 @@ class SourceOverviewServiceTest { OutputSpec.inline())); } + /** + * A policy that targets the editor: membership rides in its output metadata, not a sourceId. + */ + private void editorPolicy(String name) { + policyStore.save( + new Policy( + null, + name, + "owner", + true, + null, + List.of(), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + new OutputSpec("inline", Map.of("sources", List.of("editor"))))); + } + private void teamPolicy(String name, Long teamId, String... sourceIds) { policyStore.save( new Policy( diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index 33937b6655..29792b0162 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -7966,7 +7966,9 @@ status = "Status" usedBy = "Policies" [portal.sources.types.editor] +description = "Documents your team has processed in the editor, across policy and AI runs." label = "Editor" +noPolicies = "No policies run from the editor yet." [portal.sources.types.folder] description = "Watch a directory on the server for new documents." diff --git a/frontend/editor/src/portal/components/sources/SourceDetailCard.tsx b/frontend/editor/src/portal/components/sources/SourceDetailCard.tsx index ca659ec9d5..889a129295 100644 --- a/frontend/editor/src/portal/components/sources/SourceDetailCard.tsx +++ b/frontend/editor/src/portal/components/sources/SourceDetailCard.tsx @@ -2,7 +2,10 @@ import { useTranslation } from "react-i18next"; import { ActionIcon, Button } from "@app/ui"; import type { SourceView } from "@portal/api/sources"; import { SourceDetailPanel } from "@portal/components/sources/SourceDetailPanel"; -import { sourceTypeMeta } from "@portal/components/sources/sourceTypes"; +import { + EDITOR_SOURCE_TYPE, + sourceTypeMeta, +} from "@portal/components/sources/sourceTypes"; import "@portal/views/Sources.css"; import CloseIcon from "@mui/icons-material/Close"; @@ -30,6 +33,8 @@ export function SourceDetailCard({ const { t } = useTranslation(); const meta = sourceTypeMeta(source.type); const paused = source.status === "disabled"; + // The editor is a built-in source: it has no instance name and can't be edited/paused/deleted. + const isEditor = source.type === EDITOR_SOURCE_TYPE; return (
@@ -40,7 +45,9 @@ export function SourceDetailCard({ {meta.icon}
-

{source.name}

+

+ {isEditor ? t(meta.labelKey) : source.name} +

{t("portal.sources.detail.subtitle", { type: t(meta.labelKey), @@ -60,32 +67,34 @@ export function SourceDetailCard({ -
- - - -
+ {!isEditor && ( +
+ + + +
+ )}
); } diff --git a/frontend/editor/src/portal/components/sources/SourceDetailPanel.tsx b/frontend/editor/src/portal/components/sources/SourceDetailPanel.tsx index f4ee827575..82fd8a5b1b 100644 --- a/frontend/editor/src/portal/components/sources/SourceDetailPanel.tsx +++ b/frontend/editor/src/portal/components/sources/SourceDetailPanel.tsx @@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next"; import { Chip, StatTile } from "@app/ui"; import type { SourceView } from "@portal/api/sources"; import { Sparkline } from "@portal/components/sources/Sparkline"; +import { EDITOR_SOURCE_TYPE } from "@portal/components/sources/sourceTypes"; import "@portal/views/Sources.css"; interface SourceDetailPanelProps { @@ -20,8 +21,15 @@ export function SourceDetailPanel({ docSeries, }: SourceDetailPanelProps) { const { t } = useTranslation(); + const isEditor = source.type === EDITOR_SOURCE_TYPE; return (
+ {isEditor && ( +

+ {t("portal.sources.types.editor.description")} +

+ )} + {source.config.length > 0 && (
{source.config.map((row) => ( @@ -36,7 +44,11 @@ export function SourceDetailPanel({ {source.referencingPolicies.length === 0 ? (

- {t("portal.sources.detail.notReferenced")} + {t( + isEditor + ? "portal.sources.types.editor.noPolicies" + : "portal.sources.detail.notReferenced", + )}

) : (
diff --git a/frontend/editor/src/portal/components/sources/SourcesTable.tsx b/frontend/editor/src/portal/components/sources/SourcesTable.tsx index be66fa350e..dc6daf16aa 100644 --- a/frontend/editor/src/portal/components/sources/SourcesTable.tsx +++ b/frontend/editor/src/portal/components/sources/SourcesTable.tsx @@ -8,7 +8,10 @@ import { type TableColumn, } from "@app/ui"; import type { SourceStatus, SourceView } from "@portal/api/sources"; -import { sourceTypeMeta } from "@portal/components/sources/sourceTypes"; +import { + EDITOR_SOURCE_TYPE, + sourceTypeMeta, +} from "@portal/components/sources/sourceTypes"; import "@portal/views/Sources.css"; const STATUS_TONE: Record = { @@ -37,6 +40,9 @@ export function SourcesTable({ header: t("portal.sources.table.source"), render: (s) => { const meta = sourceTypeMeta(s.type); + // The editor is a system source with no instance name: label it from its type and drop + // the chip, which would just repeat the name. + const isEditor = s.type === EDITOR_SOURCE_TYPE; return (
- {s.name} - - {t(meta.labelKey)} - + {isEditor ? t(meta.labelKey) : s.name} + {!isEditor && ( + + {t(meta.labelKey)} + + )}
); diff --git a/frontend/editor/src/portal/components/sources/sourceTypes.ts b/frontend/editor/src/portal/components/sources/sourceTypes.ts index fb6057a183..a3f19b6b36 100644 --- a/frontend/editor/src/portal/components/sources/sourceTypes.ts +++ b/frontend/editor/src/portal/components/sources/sourceTypes.ts @@ -15,6 +15,13 @@ export interface SourceTypeMeta { accent: ChipAccent; } +/** + * The built-in editor source. It is virtual (never created/edited/deleted like a folder), always + * present in the list, and only tracks throughput - so rows of this type render without a config, + * a type chip, or edit/pause/delete actions. + */ +export const EDITOR_SOURCE_TYPE = "editor"; + const SOURCE_TYPE_META: Record = { folder: { labelKey: "portal.sources.types.folder.label", diff --git a/frontend/editor/src/portal/mocks/handlers/sources.ts b/frontend/editor/src/portal/mocks/handlers/sources.ts index 4839c17720..4f4dd0e807 100644 --- a/frontend/editor/src/portal/mocks/handlers/sources.ts +++ b/frontend/editor/src/portal/mocks/handlers/sources.ts @@ -144,14 +144,37 @@ function buildKpis(views: SourceView[]): SourceKpi[] { ]; } +/** The built-in editor source's demo throughput, mirroring the backend's per-team counters. */ +const editorDocs = { total: 8230, last24h: 42, last30d: 1680 }; + +/** The always-present editor row, pinned first and never editable/deletable. */ +function editorView(): SourceView { + return { + id: "editor", + name: "Editor", + type: "editor", + status: "active", + referenceCount: 2, + referencingPolicies: [ + { id: "pol_security", name: "Security Policy" }, + { id: "pol_redaction", name: "Redaction Policy" }, + ], + config: [], + docsTotal: editorDocs.total, + docs24h: editorDocs.last24h, + docs30d: editorDocs.last30d, + }; +} + function buildOverview(): SourcesResponse { - const views = store + const persisted = store .map((s) => toSourceView(s, refsFor(s.id))) .sort( (a, b) => b.referenceCount - a.referenceCount || a.name.localeCompare(b.name), ); - return { kpis: buildKpis(views), sources: views }; + // KPIs describe the configured connections; the built-in editor is pinned first but not counted. + return { kpis: buildKpis(persisted), sources: [editorView(), ...persisted] }; } export const sourcesHandlers = [ @@ -169,6 +192,9 @@ export const sourcesHandlers = [ http.get("/api/v1/sources/:id/document-counts", async ({ params }) => { await delay(120); + if (params.id === "editor") { + return HttpResponse.json(sampleDailySeries(editorDocs.last30d / 30)); + } const source = store.find((s) => s.id === params.id); if (!source) return new HttpResponse(null, { status: 404 }); return HttpResponse.json(docsFor(source.id).daily); @@ -195,6 +221,12 @@ export const sourcesHandlers = [ http.delete("/api/v1/sources/:id", async ({ params }) => { await delay(120); const id = String(params.id); + if (id === "editor") { + return HttpResponse.json( + { detail: "The editor is a built-in source and cannot be deleted" }, + { status: 400 }, + ); + } const source = store.find((s) => s.id === id); if (!source) return new HttpResponse(null, { status: 404 }); const refs = refsFor(id); diff --git a/frontend/editor/src/portal/views/PipelineBuilder.tsx b/frontend/editor/src/portal/views/PipelineBuilder.tsx index e82f0b380c..3b4c7d421d 100644 --- a/frontend/editor/src/portal/views/PipelineBuilder.tsx +++ b/frontend/editor/src/portal/views/PipelineBuilder.tsx @@ -51,6 +51,7 @@ import { import { clearProcessedHistory } from "@portal/api/policies"; import { availableOutputModes } from "@portal/components/pipelines/outputModes"; import { fetchSources, type SourceView } from "@portal/api/sources"; +import { EDITOR_SOURCE_TYPE } from "@portal/components/sources/sourceTypes"; import { useAsync } from "@portal/hooks/useAsync"; import { VIEW_PATHS, toPortalPath } from "@portal/contexts/ViewContext"; import { humanizeOperation } from "@portal/components/pipelines/pipelineOperations"; @@ -172,7 +173,12 @@ export function PipelineBuilder() { [id], ); const sourcesState = useAsync( - async () => (await fetchSources()).sources, + // The editor is a built-in, client-driven source (it runs on editor upload, not as a pipeline + // input), so it's excluded from the sources a pipeline can pull from. + async () => + (await fetchSources()).sources.filter( + (source) => source.type !== EDITOR_SOURCE_TYPE, + ), [], ); const triggersState = useAsync( diff --git a/frontend/editor/src/portal/views/Sources.test.tsx b/frontend/editor/src/portal/views/Sources.test.tsx index 29b38f58a9..5e281d5604 100644 --- a/frontend/editor/src/portal/views/Sources.test.tsx +++ b/frontend/editor/src/portal/views/Sources.test.tsx @@ -122,6 +122,39 @@ describe("Sources view", () => { ).toBeInTheDocument(); }); + it("shows the editor as a built-in source with no edit, pause, or delete actions", async () => { + fetchSources.mockResolvedValue({ + kpis: [], + sources: [ + { + id: "editor", + name: "Editor", + type: "editor", + status: "active", + referenceCount: 1, + referencingPolicies: [{ id: "pol-1", name: "Redaction" }], + config: [], + docsTotal: 8230, + docs24h: 42, + docs30d: 1680, + }, + ], + } satisfies SourcesResponse); + + renderView(); + + // The editor row is labelled from its type (i18n keys are returned verbatim here). + fireEvent.click( + await screen.findByText("portal.sources.types.editor.label"), + ); + + // Detail opens, but none of the mutate actions are offered for the built-in source. + await screen.findByText("portal.sources.detail.documents"); + expect(screen.queryByText("portal.sources.detail.edit")).toBeNull(); + expect(screen.queryByText("portal.sources.detail.pause")).toBeNull(); + expect(screen.queryByText("portal.sources.detail.delete")).toBeNull(); + }); + it("pauses a source by re-saving it with enabled flipped off", async () => { fetchSources.mockResolvedValue(RESPONSE); fetchSource.mockResolvedValue({