diff --git a/.github/workflows/PR-Auto-Deploy-V2.yml b/.github/workflows/PR-Auto-Deploy-V2.yml index 31135cfb4b..173b4db9df 100644 --- a/.github/workflows/PR-Auto-Deploy-V2.yml +++ b/.github/workflows/PR-Auto-Deploy-V2.yml @@ -342,9 +342,70 @@ jobs: # Set port for output echo "v2_port=${V2_PORT}" >> $GITHUB_OUTPUT + # ---- Storybook preview (only when this PR touches stories/.storybook) ---- + # Runs inside the same approved-contributor-gated deploy job, so it deploys + # under the exact same access rules as the app preview. + - name: Detect Storybook changes + id: sb-changes + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + with: + list-files: json + filters: | + storybook: + - 'frontend/**/*.stories.@(ts|tsx|mdx)' + - 'frontend/**/*.mdx' + - 'frontend/.storybook/**' + + - name: Set up Node.js for Storybook + if: steps.sb-changes.outputs.storybook == 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "22" + cache: "npm" + cache-dependency-path: frontend/package-lock.json + + - name: Install Task for Storybook + if: steps.sb-changes.outputs.storybook == 'true' + uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0 + + - name: Build and deploy Storybook + id: storybook + if: steps.sb-changes.outputs.storybook == 'true' + env: + VPS_HOST: ${{ secrets.NEW_VPS_HOST }} + VPS_USER: ${{ secrets.NEW_VPS_USERNAME }} + run: | + set -euo pipefail + # `prepare` generates the icon set stories import (not committed). + task frontend:prepare + task frontend:storybook:build + PR=${{ needs.check-pr.outputs.pr_number }} + # Served at the ROOT of its own port so Storybook's global MSW worker + # (/mockServiceWorker.js) resolves. Port = PR + 20000 (bijective, offset + # from the app preview's bare-PR-number port). + SB_PORT=$((PR + 20000)) + DIR=/stirling/SB-PR-$PR + tar czf storybook.tgz -C frontend/storybook-static . + scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + storybook.tgz "$VPS_USER@$VPS_HOST:/tmp/storybook-$PR.tgz" + ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T \ + "$VPS_USER@$VPS_HOST" << ENDSSH + set -e + rm -rf "$DIR" && mkdir -p "$DIR" + tar xzf /tmp/storybook-$PR.tgz -C "$DIR" + rm -f /tmp/storybook-$PR.tgz + docker rm -f storybook-pr-$PR 2>/dev/null || true + docker run -d --name storybook-pr-$PR --restart unless-stopped \ + -p $SB_PORT:80 -v "$DIR":/usr/share/nginx/html:ro nginx:alpine + ENDSSH + echo "url=http://$VPS_HOST:$SB_PORT/" >> "$GITHUB_OUTPUT" + - name: Post V2 deployment URL to PR if: success() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SB_URL: ${{ steps.storybook.outputs.url }} + SB_FILES: ${{ steps.sb-changes.outputs.storybook_files }} with: github-token: ${{ steps.setup-bot.outputs.token }} script: | @@ -376,11 +437,32 @@ jobs: ? `🧩 **Admin portal** included - try it at [${deploymentUrl}/portal](${deploymentUrl}/portal).\n\n` : ``; + // Storybook preview: only present when this PR changed stories/config. + const sbUrl = process.env.SB_URL; + let storybookNote = ""; + if (sbUrl) { + const files = JSON.parse(process.env.SB_FILES || "[]"); + const stories = files.filter((f) => /\.stories\.(ts|tsx|mdx)$/.test(f)); + const config = files.filter((f) => f.startsWith("frontend/.storybook/")); + const shorten = (f) => + f.replace(/^frontend\/editor\/src\//, "").replace(/^frontend\//, ""); + const storyList = stories.map((f) => `- \`${shorten(f)}\``).join("\n"); + const configList = config.map((f) => `- \`${shorten(f)}\``).join("\n"); + const summary = + `${stories.length} stor${stories.length === 1 ? "y" : "ies"} changed` + + (config.length ? ` (+${config.length} config file${config.length === 1 ? "" : "s"})` : ""); + storybookNote = + `πŸ“š **Storybook:** [${sbUrl}](${sbUrl})\n\n` + + `
\n${summary}\n\n` + + (storyList ? `**Stories**\n${storyList}\n\n` : "") + + (configList ? `**Config**\n${configList}\n` : "") + + `
\n\n`; + } + const commentBody = `## πŸš€ V2 Auto-Deployment Complete!\n\n` + - `Your V2 PR with embedded architecture has been deployed!\n\n` + `πŸ”— **Direct Test URL (non-SSL)** [${deploymentUrl}](${deploymentUrl})\n\n` + - `πŸ” **Secure HTTPS URL**: unsupported currently\n\n` + portalNote + + storybookNote + `_This deployment will be automatically cleaned up when the PR is closed._\n\n` + `πŸ”„ **Auto-deployed** for approved V2 contributors.`; @@ -476,7 +558,11 @@ jobs: else echo "V2 PR directory not found, nothing to clean up" fi - + + # Remove this PR's Storybook preview (container + files), if any. + docker rm -f storybook-pr-${{ github.event.pull_request.number }} 2>/dev/null || true + rm -rf /stirling/SB-PR-${{ github.event.pull_request.number }} + # Clean up old unused images (older than 2 weeks) but keep recent ones for reuse docker image prune -af --filter "until=336h" --filter "label!=keep=true" || true diff --git a/AGENTS.md b/AGENTS.md index 9afdad5937..7e529e4be9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -453,6 +453,7 @@ The frontend is organized with a clear separation of concerns: - **CRITICAL**: Always update translations in `en-US` only - all other languages (including `en-GB`) are handled separately - Translation files are located in `frontend/editor/public/locales/` +- After changing any translation file, run `task pre-commit:fix` ## Important Notes diff --git a/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java b/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java index 7b74111870..aa521bf942 100644 --- a/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java +++ b/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java @@ -7,6 +7,7 @@ import java.time.format.DateTimeFormatter; import java.util.Calendar; import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDDocumentInformation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -17,6 +18,9 @@ import stirling.software.common.model.PdfMetadata; @Service public class PdfMetadataService { + /** ({@code {labels}}). Written by the classify-and-label tool. */ + public static final String CLASSIFICATION_KEY = "StirlingPDFClassification"; + private final ApplicationProperties applicationProperties; private final String stirlingPDFLabel; private final UserServiceInterface userService; @@ -177,4 +181,14 @@ public class PdfMetadataService { } pdf.getDocumentInformation().setAuthor(author); } + + /** + * Write the document classifier's JSON result into the custom Info-dictionary field {@link + * #CLASSIFICATION_KEY}, leaving all other metadata untouched. + */ + public void setClassificationMetadata(PDDocument pdf, String classificationJson) { + PDDocumentInformation info = pdf.getDocumentInformation(); + info.setCustomMetadataValue(CLASSIFICATION_KEY, classificationJson); + pdf.setDocumentInformation(info); + } } diff --git a/app/common/src/main/java/stirling/software/common/util/RequestUriUtils.java b/app/common/src/main/java/stirling/software/common/util/RequestUriUtils.java index faf71c2f97..4db9f118ec 100644 --- a/app/common/src/main/java/stirling/software/common/util/RequestUriUtils.java +++ b/app/common/src/main/java/stirling/software/common/util/RequestUriUtils.java @@ -57,12 +57,13 @@ public class RequestUriUtils { return true; } - // Admin portal SPA shell. Served publicly like the editor root so a direct - // nav / refresh to /portal loads the app (the JWT lives in localStorage, not - // a cookie, so the server can't authenticate the navigation itself). The + // Admin portal SPA shell (mounted at /processor β€” must match the frontend + // PORTAL_BASENAME). Served publicly like the editor root so a direct nav / + // refresh to /processor loads the app (the JWT lives in localStorage, not a + // cookie, so the server can't authenticate the navigation itself). The // portal gates access via its own auth gate + RequirePortalAccess, and its // data APIs stay protected, so serving the shell pre-auth is safe. - if (normalizedUri.equals("/portal") || normalizedUri.startsWith("/portal/")) { + if (normalizedUri.equals("/processor") || normalizedUri.startsWith("/processor/")) { return true; } diff --git a/app/common/src/test/java/stirling/software/common/util/RequestUriUtilsTest.java b/app/common/src/test/java/stirling/software/common/util/RequestUriUtilsTest.java index a13aeac82b..a6fefd1091 100644 --- a/app/common/src/test/java/stirling/software/common/util/RequestUriUtilsTest.java +++ b/app/common/src/test/java/stirling/software/common/util/RequestUriUtilsTest.java @@ -75,10 +75,10 @@ class RequestUriUtilsTest { @Test void testIsStaticResource_portalShell() { - // The admin portal SPA shell is served pre-auth so it's directly navigable. - assertTrue(RequestUriUtils.isStaticResource("/portal")); - assertTrue(RequestUriUtils.isStaticResource("/portal/users")); - assertTrue(RequestUriUtils.isStaticResource("/app", "/app/portal")); + // The admin portal SPA shell (/processor) is served pre-auth so it's directly navigable. + assertTrue(RequestUriUtils.isStaticResource("/processor")); + assertTrue(RequestUriUtils.isStaticResource("/processor/users")); + assertTrue(RequestUriUtils.isStaticResource("/app", "/app/processor")); } // --- isFrontendRoute tests --- diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java index f61a1ce8fe..a66b168163 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java @@ -305,6 +305,23 @@ public class GetInfoOnPDF { } } + /** + * Info-dictionary keys exposed above via typed getters; any other key in the dictionary is + * surfaced as custom metadata (e.g. the classification policy's StirlingPDFClassification + * entry). + */ + private static final java.util.Set STANDARD_INFO_KEYS = + java.util.Set.of( + "Title", + "Author", + "Subject", + "Keywords", + "Producer", + "Creator", + "CreationDate", + "ModDate", + "Trapped"); + private static ObjectNode extractMetadata(PDDocument document) { ObjectNode metadata = objectMapper.createObjectNode(); @@ -335,6 +352,18 @@ public class GetInfoOnPDF { if (modificationDate != null) { metadata.put("ModificationDate", modificationDate); } + + // Surface custom Info-dictionary entries (anything beyond the + // standard fields above) β€” e.g. StirlingPDFClassification + for (String key : info.getMetadataKeys()) { + if (STANDARD_INFO_KEYS.contains(key)) { + continue; + } + String value = info.getCustomMetadataValue(key); + if (value != null && !value.isBlank()) { + metadata.put(key, value); + } + } } } catch (Exception e) { log.error("Error extracting metadata: {}", e.getMessage()); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/config/AccessConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/config/AccessConfig.java index b2d6d3d12d..d763c146ee 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/config/AccessConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/config/AccessConfig.java @@ -4,7 +4,9 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import stirling.software.proprietary.access.service.DefaultPrincipalResolver; import stirling.software.proprietary.access.service.DefaultTeamLeadLookup; +import stirling.software.proprietary.access.service.PrincipalResolver; import stirling.software.proprietary.access.service.TeamLeadLookup; /** Access-layer bean wiring. */ @@ -17,4 +19,11 @@ public class AccessConfig { TeamLeadLookup defaultTeamLeadLookup() { return new DefaultTeamLeadLookup(); } + + /** USER/TEAM projection unless another bean is defined (e.g. the saas resolver). */ + @Bean + @ConditionalOnMissingBean(PrincipalResolver.class) + PrincipalResolver defaultPrincipalResolver() { + return new DefaultPrincipalResolver(); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/controller/ResourceGrantController.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/controller/ResourceGrantController.java index 5b69c6878e..cc66020670 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/controller/ResourceGrantController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/controller/ResourceGrantController.java @@ -25,7 +25,9 @@ import stirling.software.proprietary.access.model.PrincipalType; import stirling.software.proprietary.access.model.ResourceGrant; import stirling.software.proprietary.access.model.ResourceType; import stirling.software.proprietary.access.service.ResourceAccessService; +import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamRepository; /** Admin endpoints to grant/revoke access to gated resources (the portal, integration configs). */ @RestController @@ -36,6 +38,8 @@ import stirling.software.proprietary.security.model.User; public class ResourceGrantController { private final ResourceAccessService accessService; + private final UserRepository userRepository; + private final TeamRepository teamRepository; @GetMapping("/grants") public ResponseEntity list( @@ -45,6 +49,14 @@ public class ResourceGrantController { return ResponseEntity.ok(grants.stream().map(this::toDto).toList()); } + @GetMapping("/grants/by-principal") + public ResponseEntity listByPrincipal( + @RequestParam PrincipalType principalType, @RequestParam Long principalId) { + List grants = + accessService.listGrantsForPrincipal(principalType, principalId); + return ResponseEntity.ok(grants.stream().map(this::toDto).toList()); + } + @PostMapping("/grants") public ResponseEntity create( @RequestBody GrantRequest request, @AuthenticationPrincipal User admin) { @@ -57,17 +69,26 @@ public class ResourceGrantController { "error", "resourceType, principalType and principalId are required")); } + // PORTAL is a singleton (empty resourceId); every other type must name a resource. + boolean portal = request.resourceType() == ResourceType.PORTAL; + if (!portal && (request.resourceId() == null || request.resourceId().isBlank())) { + return ResponseEntity.badRequest() + .body(Map.of("error", "resourceId is required for " + request.resourceType())); + } + Long principalId = request.principalId(); + String principalError = validatePrincipalExists(request.principalType(), principalId); + if (principalError != null) { + return ResponseEntity.badRequest().body(Map.of("error", principalError)); + } AccessPermission permission = request.permission() == null ? AccessPermission.USE : request.permission(); - // PORTAL is a singleton resource; its grants always target the whole type. - String resourceId = - request.resourceType() == ResourceType.PORTAL ? "" : request.resourceId(); + String resourceId = portal ? "" : request.resourceId(); ResourceGrant grant = accessService.grant( request.resourceType(), resourceId, request.principalType(), - request.principalId(), + principalId, permission, admin); return ResponseEntity.ok(toDto(grant)); @@ -79,6 +100,14 @@ public class ResourceGrantController { return ResponseEntity.ok(Map.of("message", "Grant revoked")); } + // Rejects grants to nonexistent principals (dead rows otherwise). + private String validatePrincipalExists(PrincipalType type, Long id) { + return switch (type) { + case USER -> userRepository.existsById(id) ? null : "User " + id + " does not exist"; + case TEAM -> teamRepository.existsById(id) ? null : "Team " + id + " does not exist"; + }; + } + private Map toDto(ResourceGrant g) { Map m = new HashMap<>(); m.put("id", g.getId()); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/model/OwnedResource.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/model/OwnedResource.java index 1bca14a420..46d6e17602 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/model/OwnedResource.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/model/OwnedResource.java @@ -54,4 +54,15 @@ public abstract class OwnedResource { public Long getOwnerTeamId() { return ownerTeam != null ? ownerTeam.getId() : null; } + + /** Owner as a principal ref; null when server-owned (admin-only ownership). */ + public PrincipalRef getOwnerRef() { + if (getOwnerUserId() != null) { + return PrincipalRef.user(getOwnerUserId()); + } + if (getOwnerTeamId() != null) { + return PrincipalRef.team(getOwnerTeamId()); + } + return null; + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalRef.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalRef.java new file mode 100644 index 0000000000..e66da7a435 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalRef.java @@ -0,0 +1,20 @@ +package stirling.software.proprietary.access.model; + +import java.util.Locale; + +/** A (type, id) principal pair; the atom grants and ownership are expressed in. */ +public record PrincipalRef(PrincipalType type, Long id) { + + public static PrincipalRef user(Long id) { + return new PrincipalRef(PrincipalType.USER, id); + } + + public static PrincipalRef team(Long id) { + return new PrincipalRef(PrincipalType.TEAM, id); + } + + /** Canonical engine wire form, e.g. "user:12". */ + public String token() { + return type.name().toLowerCase(Locale.ROOT) + ":" + id; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalType.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalType.java index 9019b32845..b29b1df4c4 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalType.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/model/PrincipalType.java @@ -1,6 +1,6 @@ package stirling.software.proprietary.access.model; -/** Who a {@link ResourceGrant} is granted to. Org-wide access is expressed via default policy. */ +/** Who a {@link ResourceGrant} is granted to. */ public enum PrincipalType { USER, TEAM diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/repository/ResourceGrantRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/repository/ResourceGrantRepository.java index 76209c1abe..654a8e0854 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/repository/ResourceGrantRepository.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/repository/ResourceGrantRepository.java @@ -3,11 +3,15 @@ package stirling.software.proprietary.access.repository; import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import stirling.software.proprietary.access.model.PrincipalType; import stirling.software.proprietary.access.model.ResourceGrant; import stirling.software.proprietary.access.model.ResourceType; +import stirling.software.proprietary.security.model.User; @Repository public interface ResourceGrantRepository extends JpaRepository { @@ -18,8 +22,20 @@ public interface ResourceGrantRepository extends JpaRepository findByResourceTypeAndPrincipalTypeAndPrincipalId( ResourceType resourceType, PrincipalType principalType, Long principalId); + /** All grants held by a principal, across resource types (for the manage-access view). */ + List findByPrincipalTypeAndPrincipalId( + PrincipalType principalType, Long principalId); + void deleteByResourceTypeAndResourceId(ResourceType resourceType, String resourceId); + /** Removes every grant held by a principal; used when the user/team behind it is deleted. */ + void deleteByPrincipalTypeAndPrincipalId(PrincipalType principalType, Long principalId); + + // Detach issued grants so deleting the granting user does not hit the FK. + @Modifying + @Query("update ResourceGrant g set g.grantedBy = null where g.grantedBy = :user") + void clearGrantedBy(@Param("user") User user); + boolean existsByResourceTypeAndResourceIdAndPrincipalTypeAndPrincipalId( ResourceType resourceType, String resourceId, diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/security/ResourceAccessSecurity.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/security/ResourceAccessSecurity.java index 5189b703a1..0f35965ad4 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/security/ResourceAccessSecurity.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/security/ResourceAccessSecurity.java @@ -11,7 +11,12 @@ import stirling.software.proprietary.access.service.ResourceAccessService; import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.security.service.UserService; -/** {@code @PreAuthorize} bean for portal-access checks. Active in self-hosted and saas. */ +/** + * {@code @PreAuthorize} bean for portal-access checks. Active in self-hosted and saas. Convention: + * every portal-exclusive endpoint is gated with + * {@code @PreAuthorize("@resourceAccess.canUsePortal()")}; endpoints shared with the editor (e.g. + * the policies API) must NOT be. + */ @Component("resourceAccess") @RequiredArgsConstructor public class ResourceAccessSecurity { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/DefaultPrincipalResolver.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/DefaultPrincipalResolver.java new file mode 100644 index 0000000000..61a70a7336 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/DefaultPrincipalResolver.java @@ -0,0 +1,32 @@ +package stirling.software.proprietary.access.service; + +import java.util.HashSet; +import java.util.Set; + +import stirling.software.proprietary.access.model.PrincipalRef; +import stirling.software.proprietary.security.model.User; + +/** + * Self-hosted projection: the user and their team. One deployment = one org, so ORG_ALL is open. + */ +public class DefaultPrincipalResolver implements PrincipalResolver { + + @Override + public Set principalsOf(User user) { + if (user == null) { + return Set.of(); + } + Set principals = new HashSet<>(); + principals.add(PrincipalRef.user(user.getId())); + if (user.getTeam() != null) { + principals.add(PrincipalRef.team(user.getTeam().getId())); + } + return principals; + } + + // Self-hosted is a single deployment-wide org, so ORG_ALL admits every authenticated user. + @Override + public boolean allowsDeploymentWideAccess() { + return true; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/MembershipTeamLeadLookup.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/MembershipTeamLeadLookup.java new file mode 100644 index 0000000000..9a5bece4ca --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/MembershipTeamLeadLookup.java @@ -0,0 +1,33 @@ +package stirling.software.proprietary.access.service; + +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; + +/** Real lookup backed by team_memberships LEADER rows; wins over the no-op default bean. */ +@Component +@RequiredArgsConstructor +public class MembershipTeamLeadLookup implements TeamLeadLookup { + + private final TeamMembershipRepository memberships; + + @Override + public boolean isAnyTeamLeader(User user) { + return user != null + && user.getId() != null + && memberships.existsByUserIdAndRole(user.getId(), TeamRole.LEADER); + } + + @Override + public boolean isLeaderOfTeam(User user, Long teamId) { + return user != null + && user.getId() != null + && teamId != null + && memberships.existsByTeamIdAndUserIdAndRole( + teamId, user.getId(), TeamRole.LEADER); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/OwnershipService.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/OwnershipService.java index 8db87f2c0e..5b62d8e890 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/OwnershipService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/OwnershipService.java @@ -36,15 +36,19 @@ public class OwnershipService { return accessService.canUseResource( type, String.valueOf(resource.getId()), - resource.getOwnerUserId(), + resource.getOwnerRef(), resource.getDefaultAccess(), user); } /** Whether the user may manage the resource. */ public boolean canManage(ResourceType type, OwnedResource resource, User user) { + // Disabled resources bypass grants for MANAGE too: admin/owner only. + if (!resource.isEnabled()) { + return isAdmin(user) || isOwner(resource, user); + } return accessService.canManageResource( - type, String.valueOf(resource.getId()), resource.getOwnerUserId(), user); + type, String.valueOf(resource.getId()), resource.getOwnerRef(), user); } /** diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/PrincipalResolver.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/PrincipalResolver.java new file mode 100644 index 0000000000..94fdd990da --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/PrincipalResolver.java @@ -0,0 +1,28 @@ +package stirling.software.proprietary.access.service; + +import java.util.Set; +import java.util.stream.Collectors; + +import stirling.software.proprietary.access.model.PrincipalRef; +import stirling.software.proprietary.security.model.User; + +/** Projects a user onto the set of principals they act as. */ +public interface PrincipalResolver { + + /** Every principal the user acts as; empty for a null user. */ + Set principalsOf(User user); + + /** + * Whether this deployment treats every authenticated user as one org, so the {@code ORG_ALL} + * default policy admits anyone. Self-hosted: true. Multi-tenant saas: false, so an {@code + * ORG_ALL} resource can't leak across tenants. Defaults to false (deny) for safety. + */ + default boolean allowsDeploymentWideAccess() { + return false; + } + + /** Canonical wire tokens for the engine, e.g. "user:12". */ + default Set principalTokens(User user) { + return principalsOf(user).stream().map(PrincipalRef::token).collect(Collectors.toSet()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/ResourceAccessService.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/ResourceAccessService.java index ada4c95290..72159829a4 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/ResourceAccessService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/ResourceAccessService.java @@ -14,6 +14,7 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.enumeration.Role; import stirling.software.proprietary.access.model.AccessPermission; import stirling.software.proprietary.access.model.DefaultAccessPolicy; +import stirling.software.proprietary.access.model.PrincipalRef; import stirling.software.proprietary.access.model.PrincipalType; import stirling.software.proprietary.access.model.ResourceGrant; import stirling.software.proprietary.access.model.ResourceType; @@ -29,6 +30,7 @@ public class ResourceAccessService { private final ResourceGrantRepository grantRepository; private final TeamLeadLookup teamLeadLookup; + private final PrincipalResolver principalResolver; @Value("${security.portal.defaultAccess:ADMINS_AND_TEAM_LEADS}") private DefaultAccessPolicy portalDefaultPolicy; @@ -44,28 +46,28 @@ public class ResourceAccessService { public boolean canUseResource( ResourceType type, String resourceId, - Long ownerUserId, + PrincipalRef owner, DefaultAccessPolicy defaultPolicy, User user) { if (user == null) { return false; } - if (isOwner(ownerUserId, user) || isAdmin(user)) { + if (isOwner(owner, user) || isAdmin(user)) { return true; } if (hasGrant(type, normalize(resourceId), user, AccessPermission.USE)) { return true; } - return matchesDefault(defaultPolicy, user); + return matchesDefault(defaultPolicy, owner, user); } /** Whether the user may manage (edit/delete/share) a resource. No default-policy fallback. */ public boolean canManageResource( - ResourceType type, String resourceId, Long ownerUserId, User user) { + ResourceType type, String resourceId, PrincipalRef owner, User user) { if (user == null) { return false; } - if (isOwner(ownerUserId, user) || isAdmin(user)) { + if (isOwner(owner, user) || isAdmin(user)) { return true; } return hasGrant(type, normalize(resourceId), user, AccessPermission.MANAGE); @@ -110,21 +112,22 @@ public class ResourceAccessService { return grantRepository.findByResourceTypeAndResourceId(type, normalize(resourceId)); } - /** Resource ids of the given type that this user (or their team) holds any grant on. */ + /** Every grant a principal holds, for the per-user/per-team manage-access view. */ + public List listGrantsForPrincipal( + PrincipalType principalType, Long principalId) { + return grantRepository.findByPrincipalTypeAndPrincipalId(principalType, principalId); + } + + /** Resource ids of the given type that any of the user's principals holds a grant on. */ public Set grantedResourceIds(ResourceType type, User user) { if (user == null) { return Set.of(); } Set ids = new HashSet<>(); - for (ResourceGrant g : - grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId( - type, PrincipalType.USER, user.getId())) { - ids.add(g.getResourceId()); - } - if (user.getTeam() != null) { + for (PrincipalRef principal : principalResolver.principalsOf(user)) { for (ResourceGrant g : grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId( - type, PrincipalType.TEAM, user.getTeam().getId())) { + type, principal.type(), principal.id())) { ids.add(g.getResourceId()); } } @@ -135,18 +138,12 @@ public class ResourceAccessService { private boolean hasGrant( ResourceType type, String resourceId, User user, AccessPermission required) { - Long teamId = user.getTeam() != null ? user.getTeam().getId() : null; + Set principals = principalResolver.principalsOf(user); for (ResourceGrant g : grantRepository.findByResourceTypeAndResourceId(type, resourceId)) { if (!permissionSatisfies(g.getPermission(), required)) { continue; } - if (g.getPrincipalType() == PrincipalType.USER - && g.getPrincipalId().equals(user.getId())) { - return true; - } - if (g.getPrincipalType() == PrincipalType.TEAM - && teamId != null - && g.getPrincipalId().equals(teamId)) { + if (principals.contains(new PrincipalRef(g.getPrincipalType(), g.getPrincipalId()))) { return true; } } @@ -161,20 +158,40 @@ public class ResourceAccessService { return held == AccessPermission.MANAGE; } - private boolean matchesDefault(DefaultAccessPolicy policy, User user) { + private boolean matchesDefault(DefaultAccessPolicy policy, PrincipalRef owner, User user) { if (policy == null) { return false; } return switch (policy) { - case ORG_ALL -> true; - // Admins already pass above; only team leads here. - case ADMINS_AND_TEAM_LEADS -> teamLeadLookup.isAnyTeamLeader(user); + // Deployment-wide only where the resolver treats everyone as one org; saas resolvers + // return false, so ORG_ALL cannot leak a tenant's resource to another tenant's users. + case ORG_ALL -> principalResolver.allowsDeploymentWideAccess(); + // Admins already pass above; only team leads here, scoped to the owning team. + case ADMINS_AND_TEAM_LEADS -> matchesTeamLeadDefault(owner, user); case EXPLICIT_ONLY -> false; }; } - private boolean isOwner(Long ownerUserId, User user) { - return ownerUserId != null && ownerUserId.equals(user.getId()); + // Portal (no owner) admits any team lead; a team-owned resource admits only that team's + // leads; a user-owned resource admits no extra leads. + private boolean matchesTeamLeadDefault(PrincipalRef owner, User user) { + if (owner == null) { + return teamLeadLookup.isAnyTeamLeader(user); + } + return owner.type() == PrincipalType.TEAM + && owner.id() != null + && teamLeadLookup.isLeaderOfTeam(user, owner.id()); + } + + // Team owners are the owning team's leaders; plain members are not. + private boolean isOwner(PrincipalRef owner, User user) { + if (owner == null || owner.id() == null) { + return false; + } + return switch (owner.type()) { + case USER -> owner.id().equals(user.getId()); + case TEAM -> teamLeadLookup.isLeaderOfTeam(user, owner.id()); + }; } private boolean isAdmin(User user) { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/SecretMasker.java b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/SecretMasker.java index 917c180116..f160b92759 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/access/service/SecretMasker.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/access/service/SecretMasker.java @@ -18,15 +18,26 @@ public class SecretMasker { // Cap recursion so a pathologically nested payload cannot overflow the stack. private static final int MAX_DEPTH = 32; + // Key-name substrings that mark a value sensitive. Over-masking a non-secret is + // safe; leaking a secret is not, so this errs broad - but a per-type schema + // whitelist would be a stronger boundary for free-form config (follow-up). private static final Set SENSITIVE_HINTS = Set.of( "secret", "password", + "passphrase", + "pwd", "token", "apikey", "accesskey", "credential", - "privatekey"); + "privatekey", + "authorization", + "cookie", + "session", + "connectionstring", + "bearer", + "signature"); /** Replace sensitive values with the mask (recursively) for safe display. */ public Map mask(Map config) { @@ -73,18 +84,24 @@ public class SecretMasker { private Map merge( Map stored, Map incoming, int depth) { - Map out = new LinkedHashMap<>(stored); + // Replace semantics (PUT): the result is the incoming document, except a redacted secret + // keeps its stored value. Keys absent from incoming are dropped, so edits can remove them. + Map out = new LinkedHashMap<>(); for (Map.Entry e : incoming.entrySet()) { String key = e.getKey(); Object value = e.getValue(); if (isSensitive(key)) { - if (!isRedacted(value, depth)) { + if (isRedacted(value, depth)) { + if (stored.containsKey(key)) { + out.put(key, stored.get(key)); // keep the stored secret + } + } else { out.put(key, value); // a real new secret replaces the stored one } - continue; // redacted (blank / mask) -> keep stored + continue; } if (depth < MAX_DEPTH - && out.get(key) instanceof Map s + && stored.get(key) instanceof Map s && value instanceof Map i) { out.put(key, merge(castMap(s), castMap(i), depth + 1)); } else { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationLabelsController.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationLabelsController.java new file mode 100644 index 0000000000..6dcd19e724 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationLabelsController.java @@ -0,0 +1,136 @@ +package stirling.software.proprietary.classification; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.UserServiceInterface; +import stirling.software.proprietary.classification.model.ClassificationLabels; +import stirling.software.proprietary.classification.model.LabelsValidator; +import stirling.software.proprietary.classification.store.ClassificationLabelStore; +import stirling.software.proprietary.classification.store.TeamLabelsEntity; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; + +/** + * Read/write the team's classification label set β€” the flat vocabulary the document classifier runs + * against. Shared and team-scoped exactly like policies: every user reads their own team's labels, + * and only a user who may edit policies (a team leader on SaaS, the global admin self-hosted; see + * {@link PolicyManagementAuthority}) may change it β€” gated only when login is enabled, since + * single-user deployments trust the local operator. A team with no stored labels reads as {@code + * 204}; that team has no vocabulary, so its documents are not classified (there is no built-in + * default on the backend or the engine β€” the label data lives only in the frontend). + */ +@RestController +@RequestMapping("/api/v1/classification/labels") +@Hidden +@RequiredArgsConstructor +@Tag(name = "Classification", description = "Team-scoped document-classification labels") +@ConditionalOnBooleanProperty(name = "policies.enabled") +public class ClassificationLabelsController { + + private final ClassificationLabelStore labelStore; + private final PolicyManagementAuthority policyManagementAuthority; + private final ApplicationProperties applicationProperties; + private final UserServiceInterface userService; + + @GetMapping + @Operation( + summary = "Get the team's classification labels", + description = + "Returns the caller's team label set, or 204 when the team has none (its" + + " documents are then not classified).") + public ResponseEntity getTeamLabels() { + return labelStore + .findByTeam(currentTeamId()) + .map(ResponseEntity::ok) + .orElseGet(() -> ResponseEntity.noContent().build()); + } + + @PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE) + @Operation( + summary = "Save the team's classification labels", + description = + "Validates and stores the label set for the caller's team, shared by everyone" + + " on the team. Requires the policy-editor role for the team.") + public ResponseEntity saveTeamLabels( + @RequestBody ClassificationLabels labels) { + requireEditingAllowed(); + validate(labels); + ClassificationLabels saved = labelStore.save(currentTeamId(), labels, currentUsername()); + return ResponseEntity.ok(saved); + } + + @DeleteMapping + @Operation( + summary = "Reset the team's classification labels", + description = + "Removes the team's stored label set; its documents are then not classified" + + " until labels are saved again. Requires the policy-editor role for the" + + " team.") + public ResponseEntity resetTeamLabels() { + requireEditingAllowed(); + labelStore.deleteByTeam(currentTeamId()); + return ResponseEntity.noContent().build(); + } + + private static void validate(ClassificationLabels labels) { + try { + LabelsValidator.validate(labels); + } catch (IllegalArgumentException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + } + + /** + * Editing the team labels requires the editor role for the caller's team β€” the same gate + * policies use (team leader on SaaS, global admin self-hosted). Single-user deployments (login + * disabled) have no such role, so they trust the local operator. + */ + private void requireEditingAllowed() { + if (!applicationProperties.getSecurity().isEnableLogin()) { + return; + } + if (!policyManagementAuthority.canEditPolicies()) { + throw new ResponseStatusException( + HttpStatus.FORBIDDEN, + "The team classification labels may only be changed by a team leader"); + } + } + + /** + * The caller's team key. With login disabled the single operator owns the {@link + * TeamLabelsEntity#NO_TEAM} sentinel row; with login enabled a caller with no resolvable team + * is an error rather than being dropped into the shared sentinel bucket (which would let + * unteamed users read and overwrite each other's "team" labels). + */ + private Long currentTeamId() { + Long teamId = policyManagementAuthority.currentUserTeamId(); + if (teamId != null) { + return teamId; + } + if (!applicationProperties.getSecurity().isEnableLogin()) { + return TeamLabelsEntity.NO_TEAM; + } + throw new ResponseStatusException( + HttpStatus.UNAUTHORIZED, "Could not resolve the current user's team"); + } + + private String currentUsername() { + return userService == null ? null : userService.getCurrentUsername(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabel.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabel.java new file mode 100644 index 0000000000..6601cba7e5 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabel.java @@ -0,0 +1,9 @@ +package stirling.software.proprietary.classification.model; + +/** + * One entry in the classification vocabulary. {@code id} is the label's stable identity (a slug, + * unique within a set): it is what the engine returns and what is stored on the document. {@code + * name} is the human display text the classifier model reasons over. {@code icon} is an optional + * presentational key (a Material Symbols name shown in the file sidebar); the engine never sees it. + */ +public record ClassificationLabel(String id, String name, String icon) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabels.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabels.java new file mode 100644 index 0000000000..ea7a1269f0 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabels.java @@ -0,0 +1,16 @@ +package stirling.software.proprietary.classification.model; + +import java.util.List; + +/** + * A flat multi-label classification vocabulary β€” the set of labels a document may be assigned. + * Stored per team (admin-edited, shared by everyone on the team); the classifier runs against these + * label names. A team with no stored set has no vocabulary, so its documents are not classified β€” + * neither the backend nor the engine holds a default of its own. + */ +public record ClassificationLabels(List labels) { + + public ClassificationLabels { + labels = labels == null ? List.of() : List.copyOf(labels); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/LabelsValidator.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/LabelsValidator.java new file mode 100644 index 0000000000..722e8b417b --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/LabelsValidator.java @@ -0,0 +1,71 @@ +package stirling.software.proprietary.classification.model; + +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Structural validation for a user- or admin-supplied label set, run before it is stored so a + * malformed vocabulary can never reach the classifier. Mirrors the invariants the engine relies on: + * non-blank ids and names, each unique within the set (ids exactly, names case-insensitively). + */ +public final class LabelsValidator { + + private LabelsValidator() {} + + // Generous upper bounds so a legitimate label set is never blocked, but a single team or user + // can't store an unbounded blob that would bloat the row, balloon the classifier prompt, or + // exhaust memory on deserialize. + static final int MAX_LABELS = 500; + static final int MAX_TEXT_LENGTH = 128; + + // Icon is a Material Symbols key (lowercase, digits, hyphens). Enforce the SHAPE server-side β€” + // the exact allowlist lives in the frontend β€” so a client bypassing the UI can't store + // arbitrary + // text that would render as garbage (or worse) in every teammate's sidebar. + private static final Pattern ICON_KEY = Pattern.compile("^[a-z0-9-]+$"); + + /** + * @throws IllegalArgumentException with a human-readable message when the label set is invalid. + */ + public static void validate(ClassificationLabels labels) { + if (labels == null || labels.labels() == null) { + throw new IllegalArgumentException("Labels are required"); + } + if (labels.labels().size() > MAX_LABELS) { + throw new IllegalArgumentException("Too many labels (max " + MAX_LABELS + ")"); + } + Set ids = new HashSet<>(); + Set names = new HashSet<>(); + for (ClassificationLabel label : labels.labels()) { + requireText(label.id(), "Label id"); + requireText(label.name(), "Label name"); + if (label.icon() != null && !label.icon().isEmpty()) { + if (label.icon().length() > MAX_TEXT_LENGTH) { + throw new IllegalArgumentException( + "Label icon is too long (max " + MAX_TEXT_LENGTH + " characters)"); + } + if (!ICON_KEY.matcher(label.icon()).matches()) { + throw new IllegalArgumentException("Invalid label icon: " + label.icon()); + } + } + if (!ids.add(label.id().trim())) { + throw new IllegalArgumentException("Duplicate label id: " + label.id()); + } + if (!names.add(label.name().trim().toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException("Duplicate label name: " + label.name()); + } + } + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + if (value.trim().length() > MAX_TEXT_LENGTH) { + throw new IllegalArgumentException( + field + " is too long (max " + MAX_TEXT_LENGTH + " characters)"); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/ClassificationLabelStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/ClassificationLabelStore.java new file mode 100644 index 0000000000..a604854c60 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/ClassificationLabelStore.java @@ -0,0 +1,22 @@ +package stirling.software.proprietary.classification.store; + +import java.util.Optional; + +import stirling.software.proprietary.classification.model.ClassificationLabels; + +/** + * Stores one {@link ClassificationLabels} set per team. A {@code null} teamId addresses the + * unteamed set (login disabled / no resolvable team), mirroring how the policy store treats a null + * team. + */ +public interface ClassificationLabelStore { + + /** The team's stored labels, or empty when it has none (callers then skip classification). */ + Optional findByTeam(Long teamId); + + /** Create or replace the team's labels. Returns the stored value. */ + ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy); + + /** Remove the team's labels (reset to default). Returns whether a set existed. */ + boolean deleteByTeam(Long teamId); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/InProcessClassificationLabelStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/InProcessClassificationLabelStore.java new file mode 100644 index 0000000000..6846902eab --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/InProcessClassificationLabelStore.java @@ -0,0 +1,36 @@ +package stirling.software.proprietary.classification.store; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import stirling.software.proprietary.classification.model.ClassificationLabels; + +/** + * In-memory {@link ClassificationLabelStore} for tests and any future no-database mode. {@link + * JpaClassificationLabelStore} is the runtime bean. + */ +public class InProcessClassificationLabelStore implements ClassificationLabelStore { + + private final Map byTeam = new ConcurrentHashMap<>(); + + @Override + public Optional findByTeam(Long teamId) { + return Optional.ofNullable(byTeam.get(key(teamId))); + } + + @Override + public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) { + byTeam.put(key(teamId), labels); + return labels; + } + + @Override + public boolean deleteByTeam(Long teamId) { + return byTeam.remove(key(teamId)) != null; + } + + private static long key(Long teamId) { + return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/JpaClassificationLabelStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/JpaClassificationLabelStore.java new file mode 100644 index 0000000000..724e335fc4 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/JpaClassificationLabelStore.java @@ -0,0 +1,76 @@ +package stirling.software.proprietary.classification.store; + +import java.time.Instant; +import java.util.Optional; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.classification.model.ClassificationLabels; + +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; + +/** + * Durable {@link ClassificationLabelStore} backed by JPA; the runtime store. Gated on {@code + * policies.enabled} β€” stored labels only matter when the Classification policy can run β€” so it + * shares the policy subsystem's on/off switch. Each label set is persisted as JSON via {@link + * TeamLabelsEntity}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnBooleanProperty(name = "policies.enabled") +public class JpaClassificationLabelStore implements ClassificationLabelStore { + + private final TeamLabelsRepository teamRepository; + private final ObjectMapper objectMapper; + + @Override + public Optional findByTeam(Long teamId) { + return teamRepository + .findById(key(teamId)) + .flatMap(entity -> parse(entity.getLabelsJson(), "team " + teamId)); + } + + @Override + public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) { + TeamLabelsEntity entity = new TeamLabelsEntity(); + entity.setTeamId(key(teamId)); + entity.setLabelsJson(objectMapper.writeValueAsString(labels)); + entity.setUpdatedAt(Instant.now()); + entity.setUpdatedBy(updatedBy); + teamRepository.save(entity); + return labels; + } + + @Override + public boolean deleteByTeam(Long teamId) { + long id = key(teamId); + if (!teamRepository.existsById(id)) { + return false; + } + teamRepository.deleteById(id); + return true; + } + + private Optional parse(String json, String owner) { + try { + return Optional.of(objectMapper.readValue(json, ClassificationLabels.class)); + } catch (JacksonException e) { + // A stored label set that no longer parses (corruption / manual DB edit) must not break + // classification: drop it so the caller treats the team as having no labels (and skips + // classification) rather than surfacing a 500 on every upload. + log.warn("Discarding unparseable stored labels for {}: {}", owner, e.getMessage()); + return Optional.empty(); + } + } + + /** Map the nullable team id onto the entity's non-null key (sentinel for the unteamed case). */ + private static long key(Long teamId) { + return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsEntity.java new file mode 100644 index 0000000000..25f2e5b259 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsEntity.java @@ -0,0 +1,47 @@ +package stirling.software.proprietary.classification.store; + +import java.io.Serializable; +import java.time.Instant; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * JPA row for a team's classification labels β€” one row per team. The label set lives as JSON in + * {@code labelsJson} (authoritative on read). {@code teamId} is the natural key; the sentinel + * {@link #NO_TEAM} stands in for the unteamed (login-disabled / self-hosted single-team) case, + * since a primary key can't be null (policies store a nullable {@code team_id}, but this table is + * keyed one-per-team). Kept decoupled from the security entities β€” {@code teamId} is a plain value, + * not a foreign key β€” so classification can be enabled or disabled without touching them. + */ +@Entity +@Table(name = "classification_labels") +@NoArgsConstructor +@Getter +@Setter +public class TeamLabelsEntity implements Serializable { + + private static final long serialVersionUID = 1L; + + /** Sentinel key for the unteamed label set (login disabled / no resolvable team). */ + public static final long NO_TEAM = 0L; + + @Id + @Column(name = "team_id") + private long teamId; + + @Column(name = "labels_json", columnDefinition = "text") + private String labelsJson; + + @Column(name = "updated_at") + private Instant updatedAt; + + @Column(name = "updated_by") + private String updatedBy; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsRepository.java new file mode 100644 index 0000000000..7fc9589c39 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsRepository.java @@ -0,0 +1,7 @@ +package stirling.software.proprietary.classification.store; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface TeamLabelsRepository extends JpaRepository {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java new file mode 100644 index 0000000000..629a377cf3 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java @@ -0,0 +1,218 @@ +package stirling.software.proprietary.controller.api; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import io.github.pixee.security.Filenames; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.common.service.UserServiceInterface; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.WebResponseUtils; +import stirling.software.proprietary.classification.model.ClassificationLabel; +import stirling.software.proprietary.classification.store.ClassificationLabelStore; +import stirling.software.proprietary.model.api.ai.AiPageText; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; +import stirling.software.proprietary.service.AiEngineClient; +import stirling.software.proprietary.service.PdfContentExtractor; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ObjectNode; + +/** + * Dispatchable tool that classifies a PDF and writes the result into its metadata. + * + *

Runs as a Classification-policy pipeline step: it reads a bounded page window, asks the AI + * engine to classify the document against the caller's team label set, and stores the engine's JSON + * answer β€” minus the transport-only {@code outcome} field β€” in the custom Info-dictionary key + * {@link PdfMetadataService#CLASSIFICATION_KEY}. Returns the labelled PDF. Not intended for direct + * client use. + */ +@Slf4j +@Hidden +@RestController +@RequestMapping("/api/v1/ai/tools") +@Tag(name = "AI Tools", description = "Dispatchable AI-backed tools.") +public class ClassifyLabelController { + + /** Pages read from each end of the document β€” mirrors the engine's window. */ + private static final int WINDOW_PAGES = 2; + + private static final String CLASSIFY_ENDPOINT = "/api/v1/documents/classify"; + + private final CustomPDFDocumentFactory pdfDocumentFactory; + private final TempFileManager tempFileManager; + private final PdfContentExtractor pdfContentExtractor; + private final PdfMetadataService pdfMetadataService; + private final AiEngineClient aiEngineClient; + private final ObjectMapper objectMapper; + private final UserServiceInterface userService; + + /** + * Present only when the policy subsystem is enabled ({@code policies.enabled}); the store and + * team authority are gated on it. Null otherwise, in which case there are no team labels to + * classify against and the document is passed through unlabelled. + */ + private final ClassificationLabelStore labelStore; + + private final PolicyManagementAuthority policyManagementAuthority; + + public ClassifyLabelController( + CustomPDFDocumentFactory pdfDocumentFactory, + TempFileManager tempFileManager, + PdfContentExtractor pdfContentExtractor, + PdfMetadataService pdfMetadataService, + AiEngineClient aiEngineClient, + ObjectMapper objectMapper, + @Autowired(required = false) UserServiceInterface userService, + @Autowired(required = false) ClassificationLabelStore labelStore, + @Autowired(required = false) PolicyManagementAuthority policyManagementAuthority) { + this.pdfDocumentFactory = pdfDocumentFactory; + this.tempFileManager = tempFileManager; + this.pdfContentExtractor = pdfContentExtractor; + this.pdfMetadataService = pdfMetadataService; + this.aiEngineClient = aiEngineClient; + this.objectMapper = objectMapper; + this.userService = userService; + this.labelStore = labelStore; + this.policyManagementAuthority = policyManagementAuthority; + } + + @PostMapping(value = "/classify-and-label", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Classify a PDF and label its metadata", + description = + "Reads the first two and last two pages, classifies the document via the AI" + + " engine, and stores the result in the StirlingPDFClassification" + + " metadata field. Dispatched by the Classification policy; not" + + " intended for direct client use.") + public ResponseEntity classifyAndLabel( + @RequestParam("fileInput") MultipartFile fileInput) throws IOException { + try (PDDocument document = pdfDocumentFactory.load(fileInput, true)) { + String fileName = safeFileName(fileInput.getOriginalFilename()); + + List allowed = resolveAllowedLabels(); + if (allowed.isEmpty()) { + // No vocabulary to classify against (the team stored no labels): pass the file + // through unlabelled rather than ask the engine to classify against nothing. + log.debug("[classify-and-label] {} has no team labels; skipping", fileName); + return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager); + } + + List pages = extractWindow(document); + String requestBody = + objectMapper.writeValueAsString( + new ClassifyEngineRequest(fileName, pages, allowed)); + + String userId = userService != null ? userService.getCurrentUsername() : null; + String responseJson = aiEngineClient.post(CLASSIFY_ENDPOINT, requestBody, userId); + + pdfMetadataService.setClassificationMetadata(document, toMetadataValue(responseJson)); + log.debug("[classify-and-label] labelled {} ({} window pages)", fileName, pages.size()); + + return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager); + } + } + + private List extractWindow(PDDocument document) throws IOException { + List pages = new ArrayList<>(); + for (int pageNumber : windowPageNumbers(document.getNumberOfPages(), WINDOW_PAGES)) { + String text = pdfContentExtractor.extractPageTextRaw(document, pageNumber); + if (text != null && !text.isBlank()) { + pages.add(new AiPageText(pageNumber, text)); + } + } + return pages; + } + + /** First and last {@code window} page numbers (1-based), de-duplicated and in order. */ + static List windowPageNumbers(int pageCount, int window) { + Set numbers = new LinkedHashSet<>(); + for (int page = 1; page <= Math.min(window, pageCount); page++) { + numbers.add(page); + } + for (int page = Math.max(1, pageCount - window + 1); page <= pageCount; page++) { + numbers.add(page); + } + return new ArrayList<>(numbers); + } + + /** Drop the transport-only {@code outcome} discriminator; keep the rest verbatim. */ + private String toMetadataValue(String engineResponseJson) { + JsonNode node = objectMapper.readTree(engineResponseJson); + if (node instanceof ObjectNode object) { + object.remove("outcome"); + } + return objectMapper.writeValueAsString(node); + } + + private static String safeFileName(String originalFilename) { + String name = Filenames.toSimpleFileName(originalFilename); + return (name == null || name.isBlank()) ? "classified.pdf" : name; + } + + /** + * The allowed labels for the caller's team as {@code {id, name}} pairs, de-duplicated by id. + * The engine shows the model the names and returns the ids (icons are presentational and never + * sent). Returns an empty list β€” the caller then skips classification β€” when the policy + * subsystem is disabled (no store) or the team has no stored labels. The engine holds no + * default vocabulary of its own, so a team's stored labels are the only source. + */ + private List resolveAllowedLabels() { + if (labelStore == null) { + return List.of(); + } + Long teamId = + policyManagementAuthority == null + ? null + : policyManagementAuthority.currentUserTeamId(); + + Map byId = new LinkedHashMap<>(); + labelStore.findByTeam(teamId).ifPresent(labels -> collectLabels(labels.labels(), byId)); + + return List.copyOf(byId.values()); + } + + private static void collectLabels( + List labels, Map into) { + for (ClassificationLabel label : labels) { + if (label.id() == null + || label.id().isBlank() + || label.name() == null + || label.name().isBlank()) { + continue; + } + into.putIfAbsent(label.id(), new EngineLabel(label.id(), label.name())); + } + } + + /** One allowed label sent to the engine: stable id + the name the model reasons over. */ + private record EngineLabel(String id, String name) {} + + /** Request body for the engine's {@code /api/v1/documents/classify} endpoint. */ + private record ClassifyEngineRequest( + String fileName, List pages, List labels) {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ProprietaryUIDataController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ProprietaryUIDataController.java index 161c1d7055..8db5f48174 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ProprietaryUIDataController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ProprietaryUIDataController.java @@ -5,6 +5,7 @@ import static stirling.software.common.util.ProviderUtils.validateProvider; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.ResponseEntity; @@ -28,13 +29,16 @@ import stirling.software.common.model.ApplicationProperties.Security.OAUTH2.Clie import stirling.software.common.model.ApplicationProperties.Security.SAML2; import stirling.software.common.model.FileInfo; import stirling.software.common.model.enumeration.Role; +import stirling.software.common.model.enumeration.TeamRole; import stirling.software.common.model.oauth2.GitHubProvider; import stirling.software.common.model.oauth2.GoogleProvider; import stirling.software.common.model.oauth2.KeycloakProvider; +import stirling.software.proprietary.access.service.ResourceAccessService; import stirling.software.proprietary.audit.AuditEventType; import stirling.software.proprietary.audit.AuditLevel; import stirling.software.proprietary.config.AuditConfigurationProperties; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.model.dto.TeamWithUserCountDTO; import stirling.software.proprietary.repository.PersistentAuditEventRepository; import stirling.software.proprietary.security.config.EnterpriseEndpoint; @@ -44,6 +48,7 @@ import stirling.software.proprietary.security.model.Authority; import stirling.software.proprietary.security.model.SessionEntity; import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.security.model.dto.AdminUserSummary; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.saml2.CustomSaml2AuthenticatedPrincipal; import stirling.software.proprietary.security.service.DatabaseServiceInterface; @@ -65,6 +70,7 @@ public class ProprietaryUIDataController { private final SessionPersistentRegistry sessionPersistentRegistry; private final UserRepository userRepository; private final TeamRepository teamRepository; + private final TeamMembershipRepository teamMembershipRepository; private final SessionRepository sessionRepository; private final DatabaseServiceInterface databaseService; private final boolean runningEE; @@ -73,6 +79,7 @@ public class ProprietaryUIDataController { private final PersistentAuditEventRepository auditRepository; private final MfaService mfaService; private final LoginAttemptService loginAttemptService; + private final ResourceAccessService resourceAccessService; public ProprietaryUIDataController( ApplicationProperties applicationProperties, @@ -80,6 +87,7 @@ public class ProprietaryUIDataController { SessionPersistentRegistry sessionPersistentRegistry, UserRepository userRepository, TeamRepository teamRepository, + TeamMembershipRepository teamMembershipRepository, SessionRepository sessionRepository, DatabaseServiceInterface databaseService, ObjectMapper objectMapper, @@ -87,12 +95,14 @@ public class ProprietaryUIDataController { UserLicenseSettingsService licenseSettingsService, PersistentAuditEventRepository auditRepository, MfaService mfaService, - LoginAttemptService loginAttemptService) { + LoginAttemptService loginAttemptService, + ResourceAccessService resourceAccessService) { this.applicationProperties = applicationProperties; this.auditConfig = auditConfig; this.sessionPersistentRegistry = sessionPersistentRegistry; this.userRepository = userRepository; this.teamRepository = teamRepository; + this.teamMembershipRepository = teamMembershipRepository; this.sessionRepository = sessionRepository; this.databaseService = databaseService; this.objectMapper = objectMapper; @@ -101,6 +111,7 @@ public class ProprietaryUIDataController { this.auditRepository = auditRepository; this.mfaService = mfaService; this.loginAttemptService = loginAttemptService; + this.resourceAccessService = resourceAccessService; } /** @@ -370,8 +381,11 @@ public class ProprietaryUIDataController { boolean premiumEnabled = applicationProperties.getPremium().isEnabled(); // Convert User entities to AdminUserSummary DTOs to exclude sensitive fields + Set leaderUserIds = leaderUserIds(); List userSummaries = - sortedUsers.stream().map(this::convertUserToSummary).toList(); + sortedUsers.stream() + .map(user -> convertUserToSummary(user, leaderUserIds)) + .toList(); AdminSettingsData data = new AdminSettingsData(); data.setUsers(userSummaries); @@ -390,6 +404,10 @@ public class ProprietaryUIDataController { data.setLicenseMaxUsers(licenseMaxUsers); data.setPremiumEnabled(premiumEnabled); data.setMailEnabled(applicationProperties.getMail().isEnabled()); + // Email invites need the invites toggle AND SMTP on; matches the inviteUsers precondition. + data.setEmailInvitesEnabled( + applicationProperties.getMail().isEnableInvites() + && applicationProperties.getMail().isEnabled()); data.setUserSettings(userSettings); data.setLockedUsers(loginAttemptService.getAllBlockedUsers()); @@ -468,9 +486,18 @@ public class ProprietaryUIDataController { teamLastRequest.put(teamId, lastActivity); } + Map> teamOwners = new HashMap<>(); + for (TeamMembership row : + teamMembershipRepository.findByRoleFetchingUserAndTeam(TeamRole.LEADER)) { + teamOwners + .computeIfAbsent(row.getTeam().getId(), id -> new ArrayList<>()) + .add(row.getUser().getUsername()); + } + TeamsData data = new TeamsData(); data.setTeamsWithCounts(teamsWithCounts); data.setTeamLastRequest(teamLastRequest); + data.setTeamOwners(teamOwners); return ResponseEntity.ok(data); } @@ -510,11 +537,17 @@ public class ProprietaryUIDataController { userLastRequest.put(username, lastRequest); } + Set ownerUserIds = + teamMembershipRepository.findByTeamIdAndRole(id, TeamRole.LEADER).stream() + .map(row -> row.getUser().getId()) + .collect(Collectors.toSet()); + TeamDetailsData data = new TeamDetailsData(); data.setTeam(team); data.setTeamUsers(teamUsers); data.setAvailableUsers(availableUsers); data.setUserLastRequest(userLastRequest); + data.setOwnerUserIds(ownerUserIds); return ResponseEntity.ok(data); } @@ -535,13 +568,24 @@ public class ProprietaryUIDataController { return ResponseEntity.ok(data); } + /** User ids holding a LEADER membership on any team. */ + private Set leaderUserIds() { + return teamMembershipRepository.findByRoleFetchingUserAndTeam(TeamRole.LEADER).stream() + .map(row -> row.getUser().getId()) + .collect(Collectors.toSet()); + } + /** * Convert User entity to AdminUserSummary DTO, excluding sensitive fields like password and * apiKey. */ - private AdminUserSummary convertUserToSummary(User user) { + private AdminUserSummary convertUserToSummary(User user, Set leaderUserIds) { AdminUserSummary summary = new AdminUserSummary(); summary.setId(user.getId()); + summary.setTeamLead(leaderUserIds.contains(user.getId())); + // Authoritative portal access, same call /me uses, so the roster honors the configured + // policy instead of the frontend guessing from role/team-leadership. + summary.setPortalAccess(resourceAccessService.canAccessPortal(user)); summary.setUsername(user.getUsername()); summary.setEmail(user.getUsername()); // Use username as email for consistency summary.setRoleName(user.getRoleName()); @@ -609,6 +653,7 @@ public class ProprietaryUIDataController { private int licenseMaxUsers; private boolean premiumEnabled; private boolean mailEnabled; + private boolean emailInvitesEnabled; private Map> userSettings; private List lockedUsers; } @@ -629,6 +674,7 @@ public class ProprietaryUIDataController { public static class TeamsData { private List teamsWithCounts; private Map teamLastRequest; + private Map> teamOwners; } @Data @@ -637,6 +683,7 @@ public class ProprietaryUIDataController { private List teamUsers; private List availableUsers; private Map userLastRequest; + private Set ownerUserIds; } @Data diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/controller/IntegrationConfigController.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/controller/IntegrationConfigController.java index 220c4e8d65..4e79fc5e7a 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/controller/IntegrationConfigController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/controller/IntegrationConfigController.java @@ -29,7 +29,9 @@ import stirling.software.proprietary.security.model.User; @RestController @RequestMapping("/api/v1/integrations") @RequiredArgsConstructor -@PreAuthorize("isAuthenticated()") +// Portal-exclusive: server-side portal-access boundary, not just isAuthenticated. Per-config +// ownership is still enforced in the service layer. +@PreAuthorize("@resourceAccess.canUsePortal()") @Tag(name = "Integrations", description = "Manage S3/MCP/API integration configurations") public class IntegrationConfigController { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java index 1fff2dd321..96e20f55cb 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java @@ -1,12 +1,16 @@ package stirling.software.proprietary.integration.crypto; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.security.GeneralSecurityException; import java.security.SecureRandom; import java.util.Arrays; import java.util.Base64; +import java.util.EnumSet; import javax.crypto.Cipher; import javax.crypto.KeyGenerator; @@ -74,7 +78,7 @@ public class CredentialEncryption { generator.init(256); SecretKey generated = generator.generateKey(); Files.createDirectories(path.getParent()); - Files.writeString(path, Base64.getEncoder().encodeToString(generated.getEncoded())); + writeOwnerOnly(path, Base64.getEncoder().encodeToString(generated.getEncoded())); log.warn( "Generated a new credential encryption key at {}. Back this file up: losing it" + " makes stored integration secrets unrecoverable.", @@ -85,6 +89,25 @@ public class CredentialEncryption { } } + // The master key decrypts every stored integration secret, so create it 0600 + // (owner-only) atomically. On non-POSIX filesystems (Windows) the config-dir + // ACL is the protection; we still create the file, just without POSIX perms. + private static void writeOwnerOnly(Path path, String content) throws IOException { + EnumSet ownerOnly = + EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); + try { + Files.createFile(path, PosixFilePermissions.asFileAttribute(ownerOnly)); + } catch (UnsupportedOperationException e) { + Files.createFile(path); + } + Files.writeString(path, content); + try { + Files.setPosixFilePermissions(path, ownerOnly); + } catch (UnsupportedOperationException ignored) { + // Non-POSIX filesystem: nothing to tighten here. + } + } + public static String encrypt(String plaintext) { if (plaintext == null) { return null; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/repository/IntegrationConfigRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/repository/IntegrationConfigRepository.java index 6efb45d745..30556060d4 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/repository/IntegrationConfigRepository.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/repository/IntegrationConfigRepository.java @@ -18,4 +18,13 @@ public interface IntegrationConfigRepository extends JpaRepository findByOwnerTeam(Team ownerTeam); List findByScope(OwnerScope scope); + + // Nested path: OwnedResource has a getOwnerTeamId() convenience getter but no such persistent + // attribute, so the plain "...OwnerTeamId" derivation resolves to a phantom property and throws + // UnknownPathException. The underscore forces the real ownerTeam.id association path. + boolean existsByOwnerTeam_Id(Long teamId); + + void deleteByOwnerUser(User ownerUser); + + void deleteByOwnerTeam_Id(Long teamId); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java index b384cc48e6..9e44b35523 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java @@ -16,6 +16,7 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.proprietary.access.model.DefaultAccessPolicy; import stirling.software.proprietary.access.model.OwnerScope; import stirling.software.proprietary.access.model.ResourceType; +import stirling.software.proprietary.access.repository.ResourceGrantRepository; import stirling.software.proprietary.access.service.OwnershipService; import stirling.software.proprietary.access.service.SecretMasker; import stirling.software.proprietary.integration.dto.IntegrationConfigRequest; @@ -41,6 +42,7 @@ public class IntegrationConfigService { private final IntegrationConfigRepository repository; private final OwnershipService ownership; private final SecretMasker secretMasker; + private final ResourceGrantRepository grantRepository; // ---- commands ---- @@ -49,6 +51,13 @@ public class IntegrationConfigService { OwnerScope scope = request.scope() == null ? OwnerScope.USER : request.scope(); IntegrationConfig cfg = new IntegrationConfig(); cfg.setIntegrationType(require(request.integrationType(), "integrationType")); + // S3 is infrastructure, not self-serve: no personal S3 for regular users. TEAM/SERVER + // scopes are already restricted to admins/team owners by assignOwnership. + if (cfg.getIntegrationType() == IntegrationType.S3 + && scope == OwnerScope.USER + && !ownership.isAdmin(currentUser)) { + throw forbidden("S3 connections can only be created by administrators or team owners"); + } cfg.setName(require(request.name(), "name")); cfg.setEnabled(request.enabled() == null || request.enabled()); cfg.setLocked(request.locked() != null && request.locked()); @@ -104,6 +113,8 @@ public class IntegrationConfigService { if (!ownership.canManage(TYPE, cfg, currentUser)) { throw forbidden("You cannot manage this integration"); } + // Drop grants sharing this config so they do not dangle as dead rows. + grantRepository.deleteByResourceTypeAndResourceId(TYPE, String.valueOf(cfg.getId())); repository.delete(cfg); } diff --git a/app/saas/src/main/java/stirling/software/saas/model/TeamMembership.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamMembership.java similarity index 96% rename from app/saas/src/main/java/stirling/software/saas/model/TeamMembership.java rename to app/proprietary/src/main/java/stirling/software/proprietary/model/TeamMembership.java index 4221643432..de121939bb 100644 --- a/app/saas/src/main/java/stirling/software/saas/model/TeamMembership.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamMembership.java @@ -1,4 +1,4 @@ -package stirling.software.saas.model; +package stirling.software.proprietary.model; import java.io.Serializable; import java.time.LocalDateTime; @@ -15,7 +15,6 @@ import lombok.Setter; import lombok.ToString; import stirling.software.common.model.enumeration.TeamRole; -import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.model.User; /** diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java index 57207d2bc9..c80615f03e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java @@ -21,4 +21,11 @@ public class AiWorkflowResultFile { @Schema(description = "MIME type of the file", example = "application/pdf") private String contentType; + + @Schema( + description = + "Index into the request's fileInputs that this output was derived from, or null" + + " when it has no single source (e.g. a merge, or a generated file)." + + " Lets the client replace that input in place as a new version.") + private Integer sourceIndex; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 64e6aa0523..9494317751 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestPart; @@ -47,6 +48,7 @@ import stirling.software.proprietary.policy.engine.PolicyRunHandle; import stirling.software.proprietary.policy.engine.PolicyRunRegistry; import stirling.software.proprietary.policy.engine.PolicyRunner; import stirling.software.proprietary.policy.engine.PolicyValidator; +import stirling.software.proprietary.policy.ledger.ProcessedLedger; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyInputs; @@ -86,6 +88,7 @@ public class PolicyController { private final PolicyManagementAuthority policyManagementAuthority; private final PolicyTriggerManager policyTriggerManager; private final PolicyOverviewService policyOverviewService; + private final ProcessedLedger processedLedger; private final List policyTriggers; private final ApplicationProperties applicationProperties; private final TempFileManager tempFileManager; @@ -213,6 +216,20 @@ public class PolicyController { return ResponseEntity.ok(saved); } + @PutMapping("/order") + @Operation( + summary = "Set the team's policy run order", + description = + "Persists the team-wide order policies run in, from the given ordered list of" + + " policy ids (position β†’ order). The per-trigger order shown in the UI" + + " is this one sequence filtered by trigger. Team-leader/admin only;" + + " ids outside the caller's team are ignored.") + public ResponseEntity reorderPolicies(@RequestBody List orderedPolicyIds) { + requirePolicyEditingAllowed(); + policyStore.reorder(policyAccessGuard.teamForNewPolicy(), orderedPolicyIds); + return ResponseEntity.noContent().build(); + } + /** * Every {@code sourceId} a policy references must resolve to a source in the caller's team, so * a client can neither reference a non-existent source nor reach across teams to use another @@ -337,6 +354,7 @@ public class PolicyController { boolean accessible = policyStore.get(policyId).filter(policyAccessGuard::canAccess).isPresent(); if (accessible && policyStore.delete(policyId)) { + processedLedger.clearPolicy(policyId); // Cancel any now-orphaned folder watch promptly rather than leaving the WatchKey open // until the next reconcile sweep. policyTriggerManager.notifyPoliciesChanged(); @@ -345,6 +363,25 @@ public class PolicyController { return ResponseEntity.notFound().build(); } + @DeleteMapping("/{policyId}/processed-history") + @Operation( + summary = "Clear a policy's processed-file history", + description = + "Forgets which source files this policy has already processed, so its next" + + " sweep reprocesses everything currently in its sources. Does not" + + " touch the files themselves.") + public ResponseEntity clearProcessedHistory(@PathVariable String policyId) { + requirePolicyEditingAllowed(); + // Scope to the caller's team: a policy in another team reads as not-found. + boolean accessible = + policyStore.get(policyId).filter(policyAccessGuard::canAccess).isPresent(); + if (!accessible) { + return ResponseEntity.notFound().build(); + } + processedLedger.clearPolicy(policyId); + return ResponseEntity.noContent().build(); + } + @PostMapping(value = "/{policyId}/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @Operation( summary = "Run a stored policy", diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java index ff5d73efb3..98a6f90096 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java @@ -35,6 +35,7 @@ import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyInputs; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.WaitState; +import stirling.software.proprietary.policy.output.OutputDelivery; import stirling.software.proprietary.policy.output.PolicyOutputSink; import stirling.software.proprietary.policy.progress.PolicyProgressListener; import stirling.software.proprietary.service.DownstreamEntitlementError; @@ -203,7 +204,12 @@ public class PolicyEngine { PolicyExecutionResult result = stepExecutor.execute(run.getDefinition(), inputs, listener); OutputSpec output = run.getDefinition().output(); - List outputs = sinkFor(output).deliver(runId, result.files(), output); + List outputs = + sinkFor(output) + .deliver( + new OutputDelivery(runId, run.getPolicyId()), + result.files(), + output); taskManager.setMultipleFileResults(runId, outputs); taskManager.setComplete(runId); run.complete(outputs); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java index 8bfb78410b..2d5552517c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java @@ -8,7 +8,11 @@ import tools.jackson.databind.JsonNode; /** * Result of a {@link PolicyExecutor} run. {@code files} are final temp files (not yet stored). - * {@code report}/{@code reportTool} carry the last step's structured report and its operation, or - * null if no step produced one. + * {@code origins} is parallel to {@code files}: each entry is the index into the original pipeline + * inputs that the output traces back to, or {@code null} when it has no single source (e.g. a merge + * combining several inputs, or a generated file). Callers use it to map an output back onto the + * file it came from. {@code report}/{@code reportTool} carry the last step's structured report and + * its operation, or null if no step produced one. */ -public record PolicyExecutionResult(List files, JsonNode report, String reportTool) {} +public record PolicyExecutionResult( + List files, List origins, JsonNode report, String reportTool) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java index fe88b40391..d6202b520e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java @@ -58,6 +58,11 @@ public class PolicyExecutor { // payload the tool surfaced alongside or instead of a file. private record ToolResult(List files, JsonNode report) {} + // A step's output files paired with each file's origin (the index into the original pipeline + // inputs it traces back to, or null when it has no single source). Origins compose across steps + // so the final result can be mapped back onto the files that entered the pipeline. + private record StepOutput(List files, List origins, JsonNode report) {} + /** * Run every step in order, feeding each step's output into the next. Supporting files in {@code * inputs} bind to named file fields and never enter the document stream. @@ -75,6 +80,12 @@ public class PolicyExecutor { List currentFiles = inputs.primary(); Map> supportingFiles = inputs.supportingFiles(); + // Seed each input with its own index as origin; steps carry these through so the final + // outputs can be traced back to the files that entered the pipeline. + List currentOrigins = new ArrayList<>(); + for (int k = 0; k < currentFiles.size(); k++) { + currentOrigins.add(k); + } // Last non-null report wins: the terminal step defines the output. JsonNode lastReport = null; String lastReportTool = null; @@ -87,8 +98,10 @@ public class PolicyExecutor { "Pipeline step " + (i + 1) + " has no operation"); } listener.onStepStart(i + 1, steps.size(), operation); - ToolResult stepResult = executeStep(step, currentFiles, supportingFiles); + StepOutput stepResult = + executeStep(step, currentFiles, currentOrigins, supportingFiles); currentFiles = stepResult.files(); + currentOrigins = stepResult.origins(); if (stepResult.report() != null) { lastReport = stepResult.report(); lastReportTool = operation; @@ -96,7 +109,7 @@ public class PolicyExecutor { listener.onStepComplete(i + 1, steps.size(), operation); } - return new PolicyExecutionResult(currentFiles, lastReport, lastReportTool); + return new PolicyExecutionResult(currentFiles, currentOrigins, lastReport, lastReportTool); } /** @@ -104,32 +117,50 @@ public class PolicyExecutor { * responses are unpacked so each inner file is its own result (e.g. split). For per-file * dispatch the first non-null report wins. */ - private ToolResult executeStep( + private StepOutput executeStep( PipelineStep step, List inputFiles, + List inputOrigins, Map> supportingFiles) throws IOException { requireAcceptedTypes(step.operation(), inputFiles); List files = new ArrayList<>(); + List origins = new ArrayList<>(); JsonNode report = null; if (toolMetadataService.isMultiInput(step.operation())) { + // One call over all inputs. The outputs derive from a single input only when exactly + // one entered; otherwise (a genuine merge) there is no single source. ToolResult r = callEndpoint(step, inputFiles, supportingFiles); - files.addAll(r.files()); + Integer origin = inputOrigins.size() == 1 ? inputOrigins.get(0) : null; + for (Resource file : r.files()) { + files.add(file); + origins.add(origin); + } report = r.report(); } else if (inputFiles.isEmpty()) { ToolResult r = callEndpoint(step, List.of(), supportingFiles); - files.addAll(r.files()); + for (Resource file : r.files()) { + files.add(file); + origins.add(null); + } report = r.report(); } else { - for (Resource file : inputFiles) { - ToolResult r = callEndpoint(step, List.of(file), supportingFiles); - files.addAll(r.files()); + // One call per file: every output of this call inherits that input's origin, so a 1:1 + // op keeps its chain and a split (one input, many outputs) tags each output with the + // same source. + for (int k = 0; k < inputFiles.size(); k++) { + Integer origin = inputOrigins.get(k); + ToolResult r = callEndpoint(step, List.of(inputFiles.get(k)), supportingFiles); + for (Resource file : r.files()) { + files.add(file); + origins.add(origin); + } if (report == null) { report = r.report(); } } } - return new ToolResult(files, report); + return new StepOutput(files, origins, report); } /** diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java index fa2a190e8f..5a35b879c0 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java @@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.proprietary.policy.input.InputSource; import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.ledger.ProcessedLedger; import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.Policy; @@ -27,7 +28,8 @@ import stirling.software.proprietary.policy.source.SourceStore; /** * Turns a policy's referenced sources into runs: each {@code sourceId} is resolved live to its * persisted {@link Source}, then to an {@link InputSpec}. Triggers decide when and call - * {@link #run(Policy)}; the controller uses the supplied-input and ad-hoc entry points. + * {@link #run(Policy)}; the controller uses the supplied-input and ad-hoc entry points. A {@link + * SweepKind#FULL} sweep also reconciles the processed-file ledger against what is present. */ @Slf4j @Service @@ -39,6 +41,12 @@ public class PolicyRunner { private final List inputSources; private final SourceStore sourceStore; private final SourceDocCounter docCounter; + private final ProcessedLedger processedLedger; + + /** Full-listing sweep: resolve every source, then reconcile the ledger. */ + public List run(Policy policy) { + return run(policy, SweepKind.FULL); + } /** * Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so @@ -47,15 +55,21 @@ public class PolicyRunner { * rest. Returns the ids of the runs it started (empty when sources yielded no work), so a * manual trigger can report back which runs to follow. */ - public List run(Policy policy) { + public List run(Policy policy, SweepKind sweep) { + long sweepStart = System.currentTimeMillis(); + PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger); + List runIds = new ArrayList<>(); List sourceIds = policy.sourceIds(); if (sourceIds.isEmpty()) { - return List.of(startRun(policy, PolicyInputs.of(List.of()), unused -> {})); + // Generator pipeline: one run with no input. Still fall through to the cleanup + // below so rows recorded for its folder outputs are pruned like anything else, + // instead of accumulating until the policy is deleted. + runIds.add(startRun(policy, PolicyInputs.of(List.of()), unused -> {})); } - List runIds = new ArrayList<>(); for (String sourceId : sourceIds) { Source source = sourceStore.get(sourceId).orElse(null); if (source == null) { + // No veto: a deleted source's rows should age out via the cleanup below. log.warn("Policy {} references missing source {}; skipping", policy.id(), sourceId); continue; } @@ -65,9 +79,21 @@ public class PolicyRunner { sourceId, source.name(), policy.id()); + // Veto: a paused source's files cannot be stamped, so they must not be pruned. + context.vetoCleanup(); continue; } - runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec())); + runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec(), context)); + } + if (context.cleanupAllowed()) { + processedLedger.markSeen(policy.id(), context.presentIdentities()); + int removed = processedLedger.deleteUnseen(policy.id(), sweepStart); + if (removed > 0) { + log.debug( + "Pruned {} ledger row(s) for files no longer present (policy {})", + removed, + policy.id()); + } } return runIds; } @@ -86,26 +112,33 @@ public class PolicyRunner { /** * Resolves the source and starts a run per unit; records how many documents the source fed and - * returns the ids of the runs started. + * returns the ids of the runs started. Any source that could not be listed completely vetoes + * this sweep's ledger cleanup. */ - private List pullAndRun(Policy policy, String sourceId, InputSpec spec) { + private List pullAndRun( + Policy policy, String sourceId, InputSpec spec, PolicySweep context) { InputSource source = sourceFor(spec); if (source == null) { log.warn( "No input source for type '{}' (policy {}); skipping", spec.type(), policy.id()); + context.vetoCleanup(); return List.of(); } + if (!source.listsExhaustively()) { + context.vetoCleanup(); + } List work; try { - work = source.resolve(spec); + work = source.resolve(spec, context); } catch (IOException | RuntimeException e) { log.warn( "Failed to resolve source '{}' for policy {}: {}", spec.type(), policy.id(), e.getMessage()); + context.vetoCleanup(); return List.of(); } List runIds = new ArrayList<>(); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java new file mode 100644 index 0000000000..5302bacdb3 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicySweep.java @@ -0,0 +1,89 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import stirling.software.proprietary.policy.input.ResolveContext; +import stirling.software.proprietary.policy.ledger.ClaimState; +import stirling.software.proprietary.policy.ledger.ProcessedFileStatus; +import stirling.software.proprietary.policy.ledger.ProcessedLedger; + +/** + * The {@link ResolveContext} for one policy sweep: scopes ledger calls to the policy, gathers the + * present-identity union across sources, prefetches claim state in bulk so per-file claims skip + * their row lookup, and vetoes presence cleanup when any source could not be listed completely + * (pruning would wrongly forget its files). + */ +final class PolicySweep implements ResolveContext { + + private final String policyId; + private final SweepKind kind; + private final ProcessedLedger ledger; + private final Set present = new HashSet<>(); + // Claim states loaded in bulk at reportPresent; a claim outside the prefetch falls back to a + // single lookup. A stale entry cannot double-claim (the ledger re-checks every transition), + // it can only defer a file to the next sweep. + private final Map prefetched = new HashMap<>(); + private final Set prefetchedIdentities = new HashSet<>(); + private boolean cleanupVetoed; + + PolicySweep(String policyId, SweepKind kind, ProcessedLedger ledger) { + this.policyId = policyId; + this.kind = kind; + this.ledger = ledger; + } + + @Override + public synchronized boolean claim(String identity, String gate, Supplier contentHash) { + ClaimState observed = + prefetchedIdentities.contains(identity) + ? prefetched.get(identity) + : ledger.statesFor(policyId, List.of(identity)).get(identity); + boolean claimed = ledger.claim(policyId, identity, gate, contentHash, observed); + if (claimed) { + // A nested source surfacing the same file later in this sweep sees it in flight + // without another lookup. + prefetchedIdentities.add(identity); + prefetched.put(identity, new ClaimState(ProcessedFileStatus.PROCESSING, gate, null)); + } + return claimed; + } + + @Override + public void settle( + String identity, String finalGate, String finalContentHash, boolean success) { + ledger.settle(policyId, identity, finalGate, finalContentHash, success); + } + + @Override + public boolean allSettledDone(String identity) { + // Deliberately not policy-scoped: consume deletion needs every claimant's consensus. + return ledger.allSettledDone(identity); + } + + @Override + public synchronized void reportPresent(Collection identities) { + if (kind == SweepKind.FULL) { + present.addAll(identities); + } + prefetched.putAll(ledger.statesFor(policyId, identities)); + prefetchedIdentities.addAll(identities); + } + + synchronized void vetoCleanup() { + cleanupVetoed = true; + } + + synchronized boolean cleanupAllowed() { + return kind == SweepKind.FULL && !cleanupVetoed; + } + + synchronized Set presentIdentities() { + return Set.copyOf(present); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/SweepKind.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/SweepKind.java new file mode 100644 index 0000000000..c2455749ae --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/SweepKind.java @@ -0,0 +1,10 @@ +package stirling.software.proprietary.policy.engine; + +/** + * How thorough a policy sweep is: {@link #FULL} (complete listing; also stamps presence and prunes + * the ledger) or {@link #LIGHT} (event-driven; claims only, cost proportional to what changed). + */ +public enum SweepKind { + FULL, + LIGHT +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java index c00f5b4a33..9270cc0c1b 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java @@ -1,12 +1,17 @@ package stirling.software.proprietary.policy.input; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Stream; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; @@ -19,17 +24,20 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.util.FileReadinessChecker; import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.ledger.FolderIdentities; import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.PolicyInputs; /** - * Reads input files from a directory; each ready file is its own unit of work so one failure does - * not affect the others. - * - *

Mode option: "consume" (default) claims each file by moving it into {@code - * .stirling/processing} then routes it to {@code .stirling/done} or {@code .stirling/error}, so - * each file runs once; "snapshot" reads without moving, so every run sees the full set. Readiness - * is checked first so files mid-write are skipped. + * Reads input files from a directory; each ready file is its own unit of work, claimed through the + * {@link ResolveContext} ledger rather than moved aside, so nothing accumulates in a work + * directory. Options: "mode" is "consume" (default: a processed file is removed once every policy + * that claimed it has settled successfully and it is still the version that ran; failures stay in + * place and are not retried until they change) or "snapshot" (stateless, every run sees the full + * set); "recursive" descends into subdirectories; "identity" is "stat" (default, any size/mtime + * change is a new version) or "hash" (content-verified, so a touch does not reprocess). Hidden + * files and directories, including the legacy {@code .stirling} work dir, are never picked up, and + * files mid-write are skipped by the readiness check. */ @Slf4j @Service @@ -38,11 +46,6 @@ import stirling.software.proprietary.policy.model.PolicyInputs; public class FolderInputSource implements InputSource { private static final String TYPE = FolderAccessGuard.FOLDER_TYPE; - // Bookkeeping lives under one hidden dir so the watched folder stays tidy. - private static final String WORK_SUBDIR = ".stirling"; - private static final String PROCESSING_SUBDIR = "processing"; - private static final String DONE_SUBDIR = "done"; - private static final String ERROR_SUBDIR = "error"; private final FileReadinessChecker readinessChecker; private final FolderAccessGuard accessGuard; @@ -68,70 +71,195 @@ public class FolderInputSource implements InputSource { } @Override - public List resolve(InputSpec spec) throws IOException { + public List resolve(InputSpec spec, ResolveContext ctx) throws IOException { FolderConfig config = FolderConfig.from(spec.options()); Path inputDir = accessGuard.requirePermitted(config.directory()); if (!Files.isDirectory(inputDir)) { - log.debug("Folder input dir does not exist: {}", inputDir); - return List.of(); + // Fail rather than return empty: an unmounted drive must read as "could not list", + // which vetoes the sweep's presence cleanup, not as "verifiably no files", which + // would wipe the policy's history and reprocess everything on remount. + throw new NoSuchFileException( + inputDir.toString(), null, "input directory does not exist"); + } + Path canonicalDir = FolderIdentities.canonicalDir(inputDir); + List present = listFiles(inputDir, config.recursive()); + + if (config.snapshot()) { + List work = new ArrayList<>(); + for (Path file : present) { + if (readinessChecker.isReady(file)) { + work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file))))); + } + } + return work; } - List ready = new ArrayList<>(); - try (Stream entries = Files.list(inputDir)) { - entries.filter(Files::isRegularFile) - .filter(readinessChecker::isReady) - .forEach(ready::add); - } + ctx.reportPresent( + present.stream() + .map(file -> FolderIdentities.identity(canonicalDir, inputDir, file)) + .toList()); List work = new ArrayList<>(); - for (Path file : ready) { - if (config.snapshot()) { - work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file))))); - } else { - Path claimed = claim(inputDir, file); - if (claimed == null) { - continue; // another sweep/process grabbed it - } - work.add( - new ResolvedInput( - PolicyInputs.of(List.of(fileResource(claimed))), - success -> route(inputDir, claimed, success))); + for (Path file : present) { + if (!readinessChecker.isReady(file)) { + continue; } + String identity = FolderIdentities.identity(canonicalDir, inputDir, file); + MemoizedContentHash contentHash = + config.hashIdentity() ? new MemoizedContentHash(file) : null; + String gate; + boolean claimed; + try { + gate = FolderIdentities.statGate(file); + claimed = ctx.claim(identity, gate, contentHash); + } catch (IOException | UncheckedIOException e) { + log.debug("Could not read {} for its version: {}", file, e.getMessage()); + continue; // vanished or unreadable mid-sweep; the next sweep sees the truth + } + if (!claimed) { + continue; + } + work.add( + new ResolvedInput( + PolicyInputs.of(List.of(fileResource(file))), + success -> + completeConsumed( + ctx, identity, file, gate, contentHash, success))); } return work; } - // Atomic move into processing/: only one sweep can win the claim, the rest see the file gone. - private Path claim(Path inputDir, Path file) { + /** + * Settle at the version this run claimed - never a re-read, so a file replaced mid-run reads as + * a new unclaimed version next sweep instead of being marked processed. Then remove the input + * only when it is still the processed version (a mid-run replacement must survive) and every + * policy that claimed it has settled DONE, so co-watching policies all read the original and + * one failure parks the file for everyone. A failed run settles ERROR and never deletes; the + * DONE row of a file that could not be deleted still stops reprocessing. + */ + private static void completeConsumed( + ResolveContext ctx, + String identity, + Path file, + String claimGate, + MemoizedContentHash contentHash, + boolean success) { + ctx.settle(identity, claimGate, claimedHash(file, claimGate, contentHash), success); + if (!success) { + return; + } try { - Path processingDir = workDir(inputDir, PROCESSING_SUBDIR); - Files.createDirectories(processingDir); - Path claimed = uniqueTarget(processingDir, file.getFileName().toString()); - Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE); - return claimed; + if (FolderIdentities.statGate(file).equals(claimGate) && ctx.allSettledDone(identity)) { + Files.deleteIfExists(file); + } + } catch (NoSuchFileException alreadyGone) { + // Removed by the user or a co-watching policy's own consensus delete: nothing to do. } catch (IOException e) { - log.debug("Could not claim {}: {}", file, e.getMessage()); + log.warn("Could not remove consumed input {}: {}", file, e.getMessage()); + } + } + + /** + * The claimed version's content hash: the value computed during the claim when the ledger + * consulted the verifier, else computed now while the file is still at the claimed gate (so the + * hash describes what actually ran), else null. Always null in stat mode. + */ + private static String claimedHash(Path file, String claimGate, MemoizedContentHash hash) { + if (hash == null) { return null; } + String computed = hash.valueIfComputed(); + if (computed != null) { + return computed; + } + try { + if (FolderIdentities.statGate(file).equals(claimGate)) { + return hash.get(); + } + } catch (IOException | UncheckedIOException e) { + log.debug("Could not hash {} at settle: {}", file, e.getMessage()); + } + return null; } - private void route(Path inputDir, Path claimed, boolean success) { - String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR; - try { - Path destDir = workDir(inputDir, subdir); - Files.createDirectories(destDir); - Files.move( - claimed, - uniqueTarget(destDir, claimed.getFileName().toString()), - StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - log.warn( - "Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage()); + /** Lazy verification tier: invoked at most once by the ledger, retained for the settle. */ + private static final class MemoizedContentHash implements Supplier { + + private final Path file; + private volatile String value; + + private MemoizedContentHash(Path file) { + this.file = file; + } + + @Override + public String get() { + if (value == null) { + try { + value = FolderIdentities.contentHash(file); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return value; + } + + String valueIfComputed() { + return value; } } - private static Path workDir(Path inputDir, String subdir) { - return inputDir.resolve(WORK_SUBDIR).resolve(subdir); + /** Every non-hidden regular file in the source, readable or not. */ + private static List listFiles(Path inputDir, boolean recursive) throws IOException { + List files = new ArrayList<>(); + if (!recursive) { + try (Stream entries = Files.list(inputDir)) { + entries.filter(Files::isRegularFile) + .filter(file -> !hidden(file)) + .forEach(files::add); + } + return files; + } + // Hidden subtrees are pruned wholesale; symlinked directories are not followed. + Files.walkFileTree( + inputDir, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory( + Path dir, BasicFileAttributes attributes) { + if (!dir.equals(inputDir) && hidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) { + if (attributes.isRegularFile() && !hidden(file)) { + files.add(file); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + log.debug("Skipping unreadable entry {}: {}", file, e.getMessage()); + return FileVisitResult.CONTINUE; + } + }); + return files; + } + + private static boolean hidden(Path path) { + Path name = path.getFileName(); + if (name != null && name.toString().startsWith(".")) { + return true; + } + try { + return Files.isHidden(path); + } catch (IOException e) { + return false; + } } private static Resource fileResource(Path path) { @@ -144,27 +272,15 @@ public class FolderInputSource implements InputSource { }; } - private static Path uniqueTarget(Path dir, String filename) { - Path candidate = dir.resolve(filename); - if (!Files.exists(candidate)) { - return candidate; - } - int dot = filename.lastIndexOf('.'); - String base = dot < 0 ? filename : filename.substring(0, dot); - String ext = dot < 0 ? "" : filename.substring(dot); - for (int n = 1; ; n++) { - Path next = dir.resolve(base + " (" + n + ")" + ext); - if (!Files.exists(next)) { - return next; - } - } - } - - record FolderConfig(Path directory, boolean snapshot) { + record FolderConfig(Path directory, boolean snapshot, boolean recursive, boolean hashIdentity) { private static final String DIRECTORY_OPTION = "directory"; private static final String MODE_OPTION = "mode"; private static final String MODE_SNAPSHOT = "snapshot"; + private static final String RECURSIVE_OPTION = "recursive"; + private static final String IDENTITY_OPTION = "identity"; + private static final String IDENTITY_STAT = "stat"; + private static final String IDENTITY_HASH = "hash"; static FolderConfig from(Map options) { Object directory = options.get(DIRECTORY_OPTION); @@ -173,7 +289,17 @@ public class FolderInputSource implements InputSource { } Object mode = options.get(MODE_OPTION); boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString()); - return new FolderConfig(Path.of(directory.toString()), snapshot); + Object recursive = options.get(RECURSIVE_OPTION); + boolean recurse = recursive != null && Boolean.parseBoolean(recursive.toString()); + Object identity = options.get(IDENTITY_OPTION); + boolean hash = identity != null && IDENTITY_HASH.equals(identity.toString()); + if (identity != null + && !IDENTITY_STAT.equals(identity.toString()) + && !IDENTITY_HASH.equals(identity.toString())) { + throw new IllegalArgumentException( + "folder input 'identity' must be 'stat' or 'hash'"); + } + return new FolderConfig(Path.of(directory.toString()), snapshot, recurse, hash); } } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java index 436f6c5263..d32c2fc546 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java @@ -24,9 +24,21 @@ public interface InputSource { /** * Resolve the spec into zero or more units of work, each carrying one run's files and a - * completion hook. Empty list means nothing to run right now. + * completion hook. Empty list means nothing to run right now. Discovery is read-only - files + * stay where the user put them; "already processed" is tracked through {@code ctx} (claim on + * pickup, settle on completion, report what is present so stale ledger rows can be pruned). */ - List resolve(InputSpec spec) throws IOException; + List resolve(InputSpec spec, ResolveContext ctx) throws IOException; + + /** + * Whether {@link #resolve} observes everything in the source (a complete listing) rather than + * e.g. only what events surfaced. Presence cleanup of the ledger is skipped for the whole + * policy unless every enabled source says true - wrongly pruning history would reprocess a + * whole folder, while keeping a few stale rows costs nothing. + */ + default boolean listsExhaustively() { + return true; + } /** * Filesystem dirs this source draws from, for the folder-watch trigger. Advisory: resolving is diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolveContext.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolveContext.java new file mode 100644 index 0000000000..9282b44d5a --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolveContext.java @@ -0,0 +1,36 @@ +package stirling.software.proprietary.policy.input; + +import java.util.Collection; +import java.util.function.Supplier; + +/** + * A source's policy-scoped window onto the processed-file ledger for one sweep. Thread-safe and + * valid for the lifetime of the work units the source issued ({@link #settle} fires from async run + * completions). + */ +public interface ResolveContext { + + /** + * Atomically claim a file at its current version; true means this sweep runs it. A null {@code + * contentHash} makes any gate change a new version; a non-null supplier is invoked at most + * once, only on a gate mismatch, and a matching hash refreshes the stored gate instead of + * reprocessing. Supplier exceptions propagate. + */ + boolean claim(String identity, String gate, Supplier contentHash); + + /** Record a claimed file's outcome at its final version ({@code finalContentHash} nullable). */ + void settle(String identity, String finalGate, String finalContentHash, boolean success); + + /** + * Whether every policy holding a ledger row for this identity has settled it DONE. Cross-policy + * by design: consume-mode deletion is a consensus of all claimants, so a shared input is + * removed only once nobody still needs it (in-flight, failed, and interrupted rows all veto). + */ + boolean allSettledDone(String identity); + + /** + * Report every identity present right now, readable or not; feeds presence cleanup of rows + * whose file is gone. + */ + void reportPresent(Collection identities); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ClaimState.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ClaimState.java new file mode 100644 index 0000000000..4f3dc0312f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ClaimState.java @@ -0,0 +1,9 @@ +package stirling.software.proprietary.policy.ledger; + +/** + * A row's claim-relevant state as read by {@link ProcessedLedger#statesFor}: what a sweep observed + * before deciding a claim. May be stale by the time the claim runs; every ledger transition + * re-checks the observed state in its WHERE clause, so staleness defers a claim to a later sweep + * rather than double-running one. + */ +public record ClaimState(ProcessedFileStatus status, String gate, String contentHash) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/FolderIdentities.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/FolderIdentities.java new file mode 100644 index 0000000000..8ece937147 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/FolderIdentities.java @@ -0,0 +1,41 @@ +package stirling.software.proprietary.policy.ledger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +import stirling.software.proprietary.billing.ContentHasher; + +/** + * The folder backend's identity and version scheme, shared by {@code FolderInputSource} and {@code + * FolderOutputSink} so outputs are recorded under exactly the identity the next scan derives. + * Directories are canonicalised with {@code toRealPath()} so symlinked aliases agree. + */ +public final class FolderIdentities { + + private FolderIdentities() {} + + /** Canonical form of a configured directory; resolves symlinks, so the dir must exist. */ + public static Path canonicalDir(Path dir) throws IOException { + return dir.toRealPath(); + } + + /** Identity of {@code file} under {@code dir}: its path re-rooted onto the canonical dir. */ + public static String identity(Path canonicalDir, Path dir, Path file) { + return canonicalDir.resolve(dir.relativize(file)).normalize().toString(); + } + + /** The cheap version gate: a change to content length or mtime means "look closer". */ + public static String statGate(Path file) throws IOException { + BasicFileAttributes attributes = Files.readAttributes(file, BasicFileAttributes.class); + return attributes.size() + ":" + attributes.lastModifiedTime().toMillis(); + } + + /** + * The strong version token: distinguishes a real change from a touch, at the cost of a read. + */ + public static String contentHash(Path file) throws IOException { + return ContentHasher.sha256(file); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/IdentityHasher.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/IdentityHasher.java new file mode 100644 index 0000000000..32c30c0254 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/IdentityHasher.java @@ -0,0 +1,19 @@ +package stirling.software.proprietary.policy.ledger; + +import java.nio.charset.StandardCharsets; + +import stirling.software.proprietary.billing.ContentHasher; + +/** + * Fixed-width key form of a source-owned identity, so any identity length fits the ledger's primary + * key. Backend-agnostic: every source type's identities are keyed through here, which is why this + * does not live with the folder backend's {@link FolderIdentities}. + */ +public final class IdentityHasher { + + private IdentityHasher() {} + + public static String identityHash(String identity) { + return ContentHasher.sha256(identity.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/InProcessProcessedLedger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/InProcessProcessedLedger.java new file mode 100644 index 0000000000..436e62f2c8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/InProcessProcessedLedger.java @@ -0,0 +1,229 @@ +package stirling.software.proprietary.policy.ledger; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * In-memory {@link ProcessedLedger} for tests and DB-less wiring; kept semantically identical to + * {@code JpaProcessedLedger} by the shared contract test. + */ +public class InProcessProcessedLedger implements ProcessedLedger { + + private final Map> rowsByPolicy = new HashMap<>(); + private final Supplier nowMillis; + + public InProcessProcessedLedger() { + this(System::currentTimeMillis); + } + + public InProcessProcessedLedger(Supplier nowMillis) { + this.nowMillis = nowMillis; + } + + @Override + public synchronized Map statesFor( + String policyId, Collection identities) { + Map rows = rowsByPolicy.getOrDefault(policyId, Map.of()); + Map states = new HashMap<>(); + for (String identity : identities) { + Row row = rows.get(identity); + if (row != null) { + states.put(identity, new ClaimState(row.status, row.gate, row.contentHash)); + } + } + return states; + } + + // Single-lock store: the live row is never staler than any observed snapshot, so decide + // against it directly; the conditional updates of the JPA ledger yield the same outcomes. + @Override + public synchronized boolean claim( + String policyId, + String identity, + String gate, + Supplier contentHash, + ClaimState observed) { + Map rows = rowsByPolicy.computeIfAbsent(policyId, key -> new HashMap<>()); + long now = nowMillis.get(); + Row row = rows.get(identity); + if (row == null) { + String hash = contentHash == null ? null : contentHash.get(); + rows.put(identity, new Row(gate, hash, ProcessedFileStatus.PROCESSING, 1, now)); + return true; + } + if (row.status == ProcessedFileStatus.PROCESSING) { + return false; + } + if (gate.equals(row.gate)) { + if (row.status == ProcessedFileStatus.INTERRUPTED && row.attempts < MAX_ATTEMPTS) { + row.status = ProcessedFileStatus.PROCESSING; + row.attempts++; + row.lastSeen = now; + return true; + } + return false; + } + if (contentHash == null) { + row.gate = gate; + row.contentHash = null; + row.status = ProcessedFileStatus.PROCESSING; + row.attempts = 1; + row.lastSeen = now; + return true; + } + String hash = contentHash.get(); + if (Objects.equals(hash, row.contentHash)) { + if (row.status == ProcessedFileStatus.INTERRUPTED && row.attempts < MAX_ATTEMPTS) { + row.gate = gate; + row.status = ProcessedFileStatus.PROCESSING; + row.attempts++; + row.lastSeen = now; + return true; + } + if (row.status != ProcessedFileStatus.INTERRUPTED) { + row.gate = gate; + row.lastSeen = now; + } + return false; + } + row.gate = gate; + row.contentHash = hash; + row.status = ProcessedFileStatus.PROCESSING; + row.attempts = 1; + row.lastSeen = now; + return true; + } + + @Override + public synchronized void settle( + String policyId, + String identity, + String finalGate, + String finalContentHash, + boolean success) { + upsertSettled( + policyId, + identity, + finalGate, + finalContentHash, + success ? ProcessedFileStatus.DONE : ProcessedFileStatus.ERROR); + } + + @Override + public synchronized void recordOutput( + String policyId, String identity, String gate, String contentHash) { + upsertSettled(policyId, identity, gate, contentHash, ProcessedFileStatus.DONE); + } + + @Override + public synchronized void forgetOutput(String policyId, String identity, String gate) { + Map rows = rowsByPolicy.get(policyId); + if (rows == null) { + return; + } + Row row = rows.get(identity); + if (row != null && row.status == ProcessedFileStatus.DONE && gate.equals(row.gate)) { + rows.remove(identity); + } + } + + private void upsertSettled( + String policyId, + String identity, + String gate, + String contentHash, + ProcessedFileStatus status) { + Map rows = rowsByPolicy.computeIfAbsent(policyId, key -> new HashMap<>()); + long now = nowMillis.get(); + Row row = rows.get(identity); + if (row == null) { + rows.put(identity, new Row(gate, contentHash, status, 1, now)); + return; + } + row.gate = gate; + row.contentHash = contentHash; + row.status = status; + row.lastSeen = now; + } + + @Override + public synchronized boolean allSettledDone(String identity) { + for (Map rows : rowsByPolicy.values()) { + Row row = rows.get(identity); + if (row != null && row.status != ProcessedFileStatus.DONE) { + return false; + } + } + return true; + } + + @Override + public synchronized void markSeen(String policyId, Collection identities) { + Map rows = rowsByPolicy.get(policyId); + if (rows == null) { + return; + } + long now = nowMillis.get(); + for (String identity : identities) { + Row row = rows.get(identity); + if (row != null) { + row.lastSeen = now; + } + } + } + + @Override + public synchronized int deleteUnseen(String policyId, long seenSinceMillis) { + Map rows = rowsByPolicy.get(policyId); + if (rows == null) { + return 0; + } + int before = rows.size(); + rows.values() + .removeIf( + row -> + row.lastSeen < seenSinceMillis + && row.status != ProcessedFileStatus.PROCESSING); + return before - rows.size(); + } + + @Override + public synchronized void clearPolicy(String policyId) { + rowsByPolicy.remove(policyId); + } + + @Override + public synchronized void recoverInterrupted() { + for (Map rows : rowsByPolicy.values()) { + for (Row row : rows.values()) { + if (row.status == ProcessedFileStatus.PROCESSING) { + row.status = ProcessedFileStatus.INTERRUPTED; + } + } + } + } + + private static final class Row { + private String gate; + private String contentHash; + private ProcessedFileStatus status; + private int attempts; + private long lastSeen; + + private Row( + String gate, + String contentHash, + ProcessedFileStatus status, + int attempts, + long lastSeen) { + this.gate = gate; + this.contentHash = contentHash; + this.status = status; + this.attempts = attempts; + this.lastSeen = lastSeen; + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/JpaProcessedLedger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/JpaProcessedLedger.java new file mode 100644 index 0000000000..43f0e6795d --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/JpaProcessedLedger.java @@ -0,0 +1,212 @@ +package stirling.software.proprietary.policy.ledger; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +/** + * Durable {@link ProcessedLedger}; the runtime bean. A fresh claim is a flushed insert so a + * concurrent winner surfaces as a constraint violation; every other transition is a conditional + * update that re-checks the observed state, so a lost race reports 0 rows and the caller skips. + * Boot recovery assumes the single node the folder-watch trigger assumes: runs live in memory, so + * after a restart every PROCESSING row is stale. + */ +@Slf4j +@Service +@ConditionalOnBooleanProperty(name = "policies.enabled") +public class JpaProcessedLedger implements ProcessedLedger { + + private static final int STAMP_CHUNK = 500; + + private final ProcessedFileRepository repository; + private final Supplier nowMillis; + + @Autowired + public JpaProcessedLedger(ProcessedFileRepository repository) { + this(repository, System::currentTimeMillis); + } + + // Clock seam so tests can pin "now"; the runtime bean uses the wall clock above. + JpaProcessedLedger(ProcessedFileRepository repository, Supplier nowMillis) { + this.repository = repository; + this.nowMillis = nowMillis; + } + + @Override + public Map statesFor(String policyId, Collection identities) { + if (identities.isEmpty()) { + return Map.of(); + } + Map identityByHash = new HashMap<>(); + for (String identity : identities) { + identityByHash.put(IdentityHasher.identityHash(identity), identity); + } + Map states = new HashMap<>(); + List hashes = List.copyOf(identityByHash.keySet()); + for (int from = 0; from < hashes.size(); from += STAMP_CHUNK) { + List rows = + repository.findByPolicyIdAndIdentityHashIn( + policyId, + hashes.subList(from, Math.min(from + STAMP_CHUNK, hashes.size()))); + for (ProcessedFileEntity row : rows) { + states.put( + identityByHash.get(row.getIdentityHash()), + new ClaimState(row.getStatus(), row.getSignature(), row.getContentHash())); + } + } + return states; + } + + @Override + public boolean claim( + String policyId, + String identity, + String gate, + Supplier contentHash, + ClaimState observed) { + String identityHash = IdentityHasher.identityHash(identity); + long now = nowMillis.get(); + if (observed == null) { + try { + repository.saveAndFlush( + new ProcessedFileEntity( + policyId, + identityHash, + identity, + gate, + contentHash == null ? null : contentHash.get(), + ProcessedFileStatus.PROCESSING, + now)); + return true; + } catch (DataIntegrityViolationException concurrentClaim) { + return false; + } + } + if (observed.status() == ProcessedFileStatus.PROCESSING) { + return false; + } + if (gate.equals(observed.gate())) { + if (observed.status() == ProcessedFileStatus.INTERRUPTED) { + return repository.retryInterruptedAtGate( + policyId, identityHash, gate, MAX_ATTEMPTS, now) + > 0; + } + return false; + } + if (contentHash == null) { + return repository.reclaimAtNewGate(policyId, identityHash, gate, now) > 0; + } + String hash = contentHash.get(); + if (hash.equals(observed.contentHash())) { + if (observed.status() == ProcessedFileStatus.INTERRUPTED) { + return repository.retryInterruptedSameContent( + policyId, identityHash, gate, hash, MAX_ATTEMPTS, now) + > 0; + } + repository.refreshGate(policyId, identityHash, gate, hash, now); + return false; + } + return repository.reclaimAtNewContent(policyId, identityHash, gate, hash, now) > 0; + } + + @Override + public void settle( + String policyId, + String identity, + String finalGate, + String finalContentHash, + boolean success) { + upsertSettled( + policyId, + identity, + finalGate, + finalContentHash, + success ? ProcessedFileStatus.DONE : ProcessedFileStatus.ERROR); + } + + @Override + public void recordOutput(String policyId, String identity, String gate, String contentHash) { + upsertSettled(policyId, identity, gate, contentHash, ProcessedFileStatus.DONE); + } + + @Override + public void forgetOutput(String policyId, String identity, String gate) { + repository.deleteDoneAt(policyId, IdentityHasher.identityHash(identity), gate); + } + + /** + * Settle-or-insert: the row may have been presence-cleaned mid-run, and an output row may be + * brand new. + */ + private void upsertSettled( + String policyId, + String identity, + String gate, + String contentHash, + ProcessedFileStatus status) { + String identityHash = IdentityHasher.identityHash(identity); + long now = nowMillis.get(); + if (repository.settle(policyId, identityHash, gate, contentHash, status, now) > 0) { + return; + } + try { + ProcessedFileEntity row = + new ProcessedFileEntity( + policyId, identityHash, identity, gate, contentHash, status, now); + repository.saveAndFlush(row); + } catch (DataIntegrityViolationException concurrentInsert) { + repository.settle(policyId, identityHash, gate, contentHash, status, now); + } + } + + @Override + public boolean allSettledDone(String identity) { + return !repository.existsByIdentityHashAndStatusNot( + IdentityHasher.identityHash(identity), ProcessedFileStatus.DONE); + } + + @Override + public void markSeen(String policyId, Collection identities) { + if (identities.isEmpty()) { + return; + } + List hashes = identities.stream().map(IdentityHasher::identityHash).toList(); + long now = nowMillis.get(); + for (int from = 0; from < hashes.size(); from += STAMP_CHUNK) { + repository.stampSeen( + policyId, + hashes.subList(from, Math.min(from + STAMP_CHUNK, hashes.size())), + now); + } + } + + @Override + public int deleteUnseen(String policyId, long seenSinceMillis) { + return repository.deleteUnseen(policyId, seenSinceMillis); + } + + @Override + public void clearPolicy(String policyId) { + repository.deleteByPolicy(policyId); + } + + @Override + @EventListener(ApplicationReadyEvent.class) + public void recoverInterrupted() { + int recovered = repository.markAllProcessingInterrupted(nowMillis.get()); + if (recovered > 0) { + log.info("Recovered {} policy input file(s) interrupted by shutdown", recovered); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java new file mode 100644 index 0000000000..2e98025aff --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java @@ -0,0 +1,106 @@ +package stirling.software.proprietary.policy.ledger; + +import java.io.Serializable; + +import org.springframework.data.domain.Persistable; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import jakarta.persistence.IdClass; +import jakarta.persistence.Index; +import jakarta.persistence.Table; +import jakarta.persistence.Transient; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * One processed-file ledger row: the version a policy last settled a file at, and where it is in + * the claim lifecycle. Keyed by SHA-256 of the source-owned identity so any identity length fits a + * fixed-width index. {@code isNew} is always true: the entity is only saved for fresh inserts + * (everything else is a conditional update), so a lost insert race surfaces as a constraint + * violation rather than a silent merge. + */ +@Entity +@Table( + name = "policy_processed_files", + indexes = { + // presence cleanup: delete this policy's rows unseen since the sweep began + @Index(name = "idx_processed_files_policy_seen", columnList = "policy_id, last_seen"), + // cross-policy deletion consensus: existsByIdentityHashAndStatusNot filters + // identity_hash on its own, so it cannot ride the (policy_id, identity_hash) PK + @Index(name = "idx_processed_files_identity", columnList = "identity_hash") + }) +@IdClass(ProcessedFileId.class) +@NoArgsConstructor +@Getter +@Setter +public class ProcessedFileEntity implements Serializable, Persistable { + + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "policy_id") + private String policyId; + + @Id + @Column(name = "identity_hash", length = 64) + private String identityHash; + + @Column(name = "identity", length = 4096) + private String identity; + + @Column(name = "signature") + private String signature; + + @Column(name = "content_hash", length = 64) + private String contentHash; + + @Enumerated(EnumType.STRING) + @Column(name = "status", length = 16) + private ProcessedFileStatus status; + + @Column(name = "attempts") + private int attempts; + + @Column(name = "last_seen") + private long lastSeen; + + @Column(name = "updated_at") + private long updatedAt; + + public ProcessedFileEntity( + String policyId, + String identityHash, + String identity, + String signature, + String contentHash, + ProcessedFileStatus status, + long nowMillis) { + this.policyId = policyId; + this.identityHash = identityHash; + this.identity = identity; + this.signature = signature; + this.contentHash = contentHash; + this.status = status; + this.attempts = 1; + this.lastSeen = nowMillis; + this.updatedAt = nowMillis; + } + + @Override + @Transient + public ProcessedFileId getId() { + return new ProcessedFileId(policyId, identityHash); + } + + @Override + @Transient + public boolean isNew() { + return true; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileId.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileId.java new file mode 100644 index 0000000000..fd3d554022 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileId.java @@ -0,0 +1,37 @@ +package stirling.software.proprietary.policy.ledger; + +import java.io.Serializable; +import java.util.Objects; + +/** Composite key for {@link ProcessedFileEntity}: one row per policy per file identity. */ +public class ProcessedFileId implements Serializable { + + private static final long serialVersionUID = 1L; + + private String policyId; + private String identityHash; + + public ProcessedFileId() {} + + public ProcessedFileId(String policyId, String identityHash) { + this.policyId = policyId; + this.identityHash = identityHash; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ProcessedFileId other)) { + return false; + } + return Objects.equals(policyId, other.policyId) + && Objects.equals(identityHash, other.identityHash); + } + + @Override + public int hashCode() { + return Objects.hash(policyId, identityHash); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileRepository.java new file mode 100644 index 0000000000..def89821ae --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileRepository.java @@ -0,0 +1,195 @@ +package stirling.software.proprietary.policy.ledger; + +import java.util.Collection; +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +/** + * Conditional updates for the processed-file ledger: each claim variant re-checks in its WHERE + * clause the state it was decided against, so a racing claim loses cleanly with 0 rows updated. + * Transactional per call so the ledger can run them without an enclosing transaction. + */ +@Repository +public interface ProcessedFileRepository + extends JpaRepository { + + /** + * Re-claim a settled row at a new gate without content verification; clears the stored hash, + * which described content this claim never checked. + */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING," + + " e.signature = :gate, e.contentHash = null, e.attempts = 1," + + " e.lastSeen = :now, e.updatedAt = :now" + + " where e.policyId = :policyId and e.identityHash = :identityHash" + + " and e.status <>" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING" + + " and e.signature <> :gate") + int reclaimAtNewGate( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate, + @Param("now") long now); + + /** Re-claim a settled row whose content verifiably changed (or was never hashed). */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING," + + " e.signature = :gate, e.contentHash = :contentHash, e.attempts = 1," + + " e.lastSeen = :now, e.updatedAt = :now" + + " where e.policyId = :policyId and e.identityHash = :identityHash" + + " and e.status <>" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING" + + " and (e.contentHash is null or e.contentHash <> :contentHash)") + int reclaimAtNewContent( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate, + @Param("contentHash") String contentHash, + @Param("now") long now); + + /** The gate moved but the content did not: track the new gate without changing status. */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.signature = :gate, e.lastSeen = :now," + + " e.updatedAt = :now" + + " where e.policyId = :policyId and e.identityHash = :identityHash" + + " and e.status <>" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING" + + " and e.contentHash = :contentHash and e.signature <> :gate") + int refreshGate( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate, + @Param("contentHash") String contentHash, + @Param("now") long now); + + /** Bounded retry of an INTERRUPTED row at the same gate. */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING," + + " e.attempts = e.attempts + 1, e.lastSeen = :now, e.updatedAt = :now" + + " where e.policyId = :policyId and e.identityHash = :identityHash" + + " and e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.INTERRUPTED" + + " and e.signature = :gate and e.attempts < :maxAttempts") + int retryInterruptedAtGate( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate, + @Param("maxAttempts") int maxAttempts, + @Param("now") long now); + + /** Bounded retry of an INTERRUPTED row whose gate moved but whose content is unchanged. */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING," + + " e.signature = :gate, e.attempts = e.attempts + 1, e.lastSeen = :now," + + " e.updatedAt = :now" + + " where e.policyId = :policyId and e.identityHash = :identityHash" + + " and e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.INTERRUPTED" + + " and e.contentHash = :contentHash and e.attempts < :maxAttempts") + int retryInterruptedSameContent( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate, + @Param("contentHash") String contentHash, + @Param("maxAttempts") int maxAttempts, + @Param("now") long now); + + /** + * Unconditional settle (only the claiming run settles a row); returns 0 when the row was + * removed mid-run so the caller re-inserts. + */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.status = :status, e.signature = :gate," + + " e.contentHash = :contentHash, e.lastSeen = :now, e.updatedAt = :now" + + " where e.policyId = :policyId and e.identityHash = :identityHash") + int settle( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate, + @Param("contentHash") String contentHash, + @Param("status") ProcessedFileStatus status, + @Param("now") long now); + + /** Whether any policy's row at this identity is in a state other than {@code status}. */ + boolean existsByIdentityHashAndStatusNot(String identityHash, ProcessedFileStatus status); + + /** One policy's rows across a chunk of identity hashes, for a sweep's claim snapshot. */ + List findByPolicyIdAndIdentityHashIn( + String policyId, Collection identityHashes); + + /** + * Remove an output record whose rename never landed, only while still settled exactly as + * recorded; a row a claim has since taken over is left alone. + */ + @Modifying + @Transactional + @Query( + "delete from ProcessedFileEntity e where e.policyId = :policyId" + + " and e.identityHash = :identityHash and e.signature = :gate" + + " and e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.DONE") + int deleteDoneAt( + @Param("policyId") String policyId, + @Param("identityHash") String identityHash, + @Param("gate") String gate); + + /** Stamp presence for the given identities; chunked by the caller for very large folders. */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.lastSeen = :now" + + " where e.policyId = :policyId and e.identityHash in :identityHashes") + int stampSeen( + @Param("policyId") String policyId, + @Param("identityHashes") Collection identityHashes, + @Param("now") long now); + + /** + * Presence cleanup: remove rows not stamped since the sweep began, keeping in-flight claims. + */ + @Modifying + @Transactional + @Query( + "delete from ProcessedFileEntity e where e.policyId = :policyId" + + " and e.lastSeen < :cutoff and e.status <>" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING") + int deleteUnseen(@Param("policyId") String policyId, @Param("cutoff") long cutoff); + + @Modifying + @Transactional + @Query("delete from ProcessedFileEntity e where e.policyId = :policyId") + int deleteByPolicy(@Param("policyId") String policyId); + + /** Boot recovery: after a restart every PROCESSING row is stale (single node). */ + @Modifying + @Transactional + @Query( + "update ProcessedFileEntity e set e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.INTERRUPTED," + + " e.updatedAt = :now" + + " where e.status =" + + " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING") + int markAllProcessingInterrupted(@Param("now") long now); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileStatus.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileStatus.java new file mode 100644 index 0000000000..68b8b5d00c --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileStatus.java @@ -0,0 +1,17 @@ +package stirling.software.proprietary.policy.ledger; + +/** Lifecycle of one {@code (policy, file)} ledger row. */ +public enum ProcessedFileStatus { + + /** Claimed; a run is in flight. */ + PROCESSING, + + /** Run completed at this version. */ + DONE, + + /** Run failed; skipped until the file changes (clear-history is the manual retry). */ + ERROR, + + /** Was PROCESSING when the JVM died; retried a bounded number of times. */ + INTERRUPTED +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedLedger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedLedger.java new file mode 100644 index 0000000000..848a764f60 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedLedger.java @@ -0,0 +1,100 @@ +package stirling.software.proprietary.policy.ledger; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Remembers which files a policy has processed, one row per {@code (policy, identity)}, so sources + * track files in place. Identities are opaque source-owned strings; versions are two-tier: a cheap + * gate compared every sweep plus an optional content hash consulted only when the gate moves. + * Presence reconciliation ({@link #markSeen} + {@link #deleteUnseen}) keeps the table bounded. + */ +public interface ProcessedLedger { + + /** + * Claims of one version before an {@link ProcessedFileStatus#INTERRUPTED} row stops retrying. + */ + int MAX_ATTEMPTS = 3; + + /** + * One-query snapshot of the rows for these identities, keyed by identity; identities with no + * row are absent. Feeds the {@code observed} parameter of {@link #claim(String, String, String, + * Supplier, ClaimState)} so a sweep decides its claims without a per-file lookup. + */ + Map statesFor(String policyId, Collection identities); + + /** + * Atomically claim a file at its current version, deciding against {@code observed} (this row's + * entry from {@link #statesFor}; null means no row was seen); true means this caller runs it. A + * stale {@code observed} cannot double-claim - every transition re-checks the observed state, + * so a lost race skips until a later sweep. A null {@code contentHash} makes any gate change a + * new version; a non-null supplier is invoked at most once, only on a gate mismatch, and a + * matching hash refreshes the stored gate instead of reprocessing. Supplier exceptions + * propagate. + */ + boolean claim( + String policyId, + String identity, + String gate, + Supplier contentHash, + ClaimState observed); + + /** Snapshot-then-claim convenience for a single file; sweeps batch via {@link #statesFor}. */ + default boolean claim( + String policyId, String identity, String gate, Supplier contentHash) { + return claim( + policyId, + identity, + gate, + contentHash, + statesFor(policyId, List.of(identity)).get(identity)); + } + + /** Record a claimed file's outcome at its final version ({@code finalContentHash} nullable). */ + void settle( + String policyId, + String identity, + String finalGate, + String finalContentHash, + boolean success); + + /** + * Record a produced file as {@link ProcessedFileStatus#DONE} so the policy skips its own + * outputs. Must be called before the file is visible at this identity; other policies have no + * row and still process it. + */ + void recordOutput(String policyId, String identity, String gate, String contentHash); + + /** + * Remove an output record whose file never became visible (its rename lost the name race to a + * concurrent writer), so whatever file actually owns that identity is claimable at any version. + * A no-op unless the row is still settled exactly as recorded, so a claim that took the row + * over in the meantime is left alone. + */ + void forgetOutput(String policyId, String identity, String gate); + + /** + * Whether every row at this identity - across all policies, by design - is {@link + * ProcessedFileStatus#DONE}. Consume-mode deletion gates on this so a shared input is removed + * only once every claimant has processed it; in-flight, failed, and interrupted rows all veto, + * parking the file. Vacuously true when no rows exist. + */ + boolean allSettledDone(String identity); + + /** Stamp presence for every identity a full-listing sweep observed. */ + void markSeen(String policyId, Collection identities); + + /** + * Remove rows not seen since {@code seenSinceMillis}, keeping in-flight claims. Only call after + * every enabled source listed completely; returns the number of rows removed. + */ + int deleteUnseen(String policyId, long seenSinceMillis); + + /** Forget everything for a policy. */ + void clearPolicy(String policyId); + + /** Boot recovery: flip stale in-flight claims to {@link ProcessedFileStatus#INTERRUPTED}. */ + void recoverInterrupted(); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java index 8821a2940b..ed3cf707ad 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java @@ -2,11 +2,18 @@ package stirling.software.proprietary.policy.output; import java.io.IOException; import java.io.InputStream; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import org.apache.commons.io.FilenameUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; @@ -19,14 +26,18 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.billing.ContentHasher; import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.ledger.FolderIdentities; +import stirling.software.proprietary.policy.ledger.ProcessedLedger; import stirling.software.proprietary.policy.model.OutputSpec; /** - * Writes a run's outputs to the {@code directory} given in the {@link OutputSpec}. Files are - * streamed (not buffered) and uniquely named to avoid clobbering. Returned {@link ResultFile}s - * carry a synthetic id since the deliverable is the file on disk, not a {@code FileStorage} entry, - * so folder outputs are not downloadable via {@code /files/{id}}. + * Writes a run's outputs to the {@code directory} given in the {@link OutputSpec}. Each output is + * staged under a hidden {@code .stirling/tmp} dir, recorded in the processed-file ledger, then + * atomically renamed into place, so the producing policy's row exists before the file is + * discoverable and half-written outputs are never visible. Returned {@link ResultFile}s carry a + * synthetic id since the deliverable is the file on disk, not a {@code FileStorage} entry. */ @Slf4j @Service @@ -37,7 +48,11 @@ public class FolderOutputSink implements PolicyOutputSink { static final String TYPE = FolderAccessGuard.FOLDER_TYPE; static final String DIRECTORY_OPTION = "directory"; + // Staging entries are renamed away within one delivery; anything older is a crash leftover. + private static final Duration STALE_TMP_AGE = Duration.ofDays(1); + private final FolderAccessGuard accessGuard; + private final ProcessedLedger processedLedger; @Override public String type() { @@ -55,20 +70,25 @@ public class FolderOutputSink implements PolicyOutputSink { } @Override - public List deliver(String runId, List outputs, OutputSpec spec) - throws IOException { + public List deliver( + OutputDelivery delivery, List outputs, OutputSpec spec) throws IOException { Path targetDir = accessGuard.requirePermitted(directoryOf(spec)); Files.createDirectories(targetDir); + Path canonicalDir = FolderIdentities.canonicalDir(targetDir); + Path tmpDir = canonicalDir.resolve(".stirling").resolve("tmp"); + Files.createDirectories(tmpDir); + sweepStaleTmp(tmpDir); List results = new ArrayList<>(); for (int i = 0; i < outputs.size(); i++) { Resource resource = outputs.get(i); String name = safeName(resource.getFilename(), i); - Path target = uniqueTarget(targetDir, name); - try (InputStream is = resource.getInputStream()) { - Files.copy(is, target); - } - long size = Files.size(target); + Path staged = tmpDir.resolve(UUID.randomUUID().toString()); + String contentHash = stage(resource, staged, delivery.policyId() != null); + long size = Files.size(staged); + // Size and mtime survive the rename. + String gate = FolderIdentities.statGate(staged); + Path target = moveIntoPlace(delivery, canonicalDir, name, staged, gate, contentHash); String contentType = MediaTypeFactory.getMediaType(name) .orElse(MediaType.APPLICATION_OCTET_STREAM) @@ -80,11 +100,95 @@ public class FolderOutputSink implements PolicyOutputSink { .contentType(contentType) .fileSize(size) .build()); - log.debug("Wrote policy run {} output to {}", runId, target); + log.debug("Wrote policy run {} output to {}", delivery.runId(), target); } return results; } + /** + * Stream the output to its staging path. For a recorded delivery (stored policy) the content + * hash is digested in the same pass, so the ledger gets both version tiers without re-reading a + * possibly huge output; ad-hoc runs record nothing and skip the digest entirely. + */ + private static String stage(Resource resource, Path staged, boolean hashed) throws IOException { + if (!hashed) { + try (InputStream is = resource.getInputStream()) { + Files.copy(is, staged); + } + return null; + } + MessageDigest digest = ContentHasher.newSha256(); + try (InputStream is = resource.getInputStream(); + DigestOutputStream out = + new DigestOutputStream(Files.newOutputStream(staged), digest)) { + is.transferTo(out); + } + return ContentHasher.toHex(digest.digest()); + } + + /** + * The ledger row must exist before the file is visible at its final path, or a sweep could + * claim the producing policy's own output in the gap. Losing the chosen name to a concurrent + * writer forgets the just-recorded row - whatever file actually owns that name must stay + * claimable at any version - then re-picks. + */ + private Path moveIntoPlace( + OutputDelivery delivery, + Path dir, + String name, + Path staged, + String gate, + String contentHash) + throws IOException { + while (true) { + Path target = uniqueTarget(dir, name); + if (delivery.policyId() != null) { + processedLedger.recordOutput( + delivery.policyId(), target.toString(), gate, contentHash); + } + try { + Files.move(staged, target, StandardCopyOption.ATOMIC_MOVE); + return target; + } catch (FileAlreadyExistsException raced) { + if (delivery.policyId() != null) { + processedLedger.forgetOutput(delivery.policyId(), target.toString(), gate); + } + log.debug("Output name {} taken concurrently; re-picking", target); + } + } + } + + /** Best-effort removal of staging leftovers from crashed deliveries. */ + private static void sweepStaleTmp(Path tmpDir) { + Instant cutoff = Instant.now().minus(STALE_TMP_AGE); + try (Stream entries = Files.list(tmpDir)) { + entries.filter(Files::isRegularFile) + .filter( + entry -> { + try { + return Files.getLastModifiedTime(entry) + .toInstant() + .isBefore(cutoff); + } catch (IOException e) { + return false; + } + }) + .forEach( + entry -> { + try { + Files.deleteIfExists(entry); + } catch (IOException e) { + log.debug( + "Could not remove stale staging file {}: {}", + entry, + e.getMessage()); + } + }); + } catch (IOException e) { + log.debug("Could not sweep staging dir {}: {}", tmpDir, e.getMessage()); + } + } + private static Path directoryOf(OutputSpec spec) { Object directory = spec.options().get(DIRECTORY_OPTION); if (directory == null || directory.toString().isBlank()) { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java index 0fcd7323ff..904eb20523 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java @@ -41,8 +41,8 @@ public class InlineOutputSink implements PolicyOutputSink { } @Override - public List deliver(String runId, List outputs, OutputSpec spec) - throws IOException { + public List deliver( + OutputDelivery delivery, List outputs, OutputSpec spec) throws IOException { List results = new ArrayList<>(); for (int i = 0; i < outputs.size(); i++) { Resource resource = outputs.get(i); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/OutputDelivery.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/OutputDelivery.java new file mode 100644 index 0000000000..ef3b59bebe --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/OutputDelivery.java @@ -0,0 +1,8 @@ +package stirling.software.proprietary.policy.output; + +/** + * Context for one run's output delivery. {@code policyId} is null for ad-hoc pipelines; when + * present, sinks record outputs in the processed-file ledger so the producing policy does not + * re-ingest them. + */ +public record OutputDelivery(String runId, String policyId) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java index 6e6c80ba4d..7a9744f9e7 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java @@ -25,6 +25,6 @@ public interface PolicyOutputSink { default void validate(OutputSpec spec) {} /** Persist/deliver the output files and return their descriptors. */ - List deliver(String runId, List outputs, OutputSpec spec) + List deliver(OutputDelivery delivery, List outputs, OutputSpec spec) throws IOException; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java index b6d882bd6d..685ddb21db 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java @@ -1,5 +1,6 @@ package stirling.software.proprietary.policy.store; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -16,6 +17,8 @@ import stirling.software.proprietary.policy.model.Policy; public class InProcessPolicyStore implements PolicyStore { private final Map policies = new ConcurrentHashMap<>(); + // Run-order position per policy id, mirroring JpaPolicyStore's sort_order column. + private final Map sortOrders = new ConcurrentHashMap<>(); @Override public Policy save(Policy policy) { @@ -35,9 +38,25 @@ public class InProcessPolicyStore implements PolicyStore { policy.output(), policy.teamId()); policies.put(id, stored); + // Existing policy keeps its position; a new one appends to the end of its team's queue. + sortOrders.computeIfAbsent(id, key -> nextSortOrder(stored.teamId())); return stored; } + private int nextSortOrder(Long teamId) { + return policies.values().stream() + .filter(policy -> Objects.equals(policy.teamId(), teamId)) + .map(policy -> sortOrders.getOrDefault(policy.id(), 0)) + .max(Comparator.naturalOrder()) + .orElse(-1) + + 1; + } + + private Comparator byRunOrder() { + return Comparator.comparingInt(policy -> sortOrders.getOrDefault(policy.id(), 0)) + .thenComparing(Policy::id); + } + @Override public Optional get(String id) { return Optional.ofNullable(policies.get(id)); @@ -45,13 +64,14 @@ public class InProcessPolicyStore implements PolicyStore { @Override public List all() { - return List.copyOf(policies.values()); + return policies.values().stream().sorted(byRunOrder()).toList(); } @Override public List findByTeam(Long teamId) { return policies.values().stream() .filter(policy -> Objects.equals(policy.teamId(), teamId)) + .sorted(byRunOrder()) .toList(); } @@ -64,8 +84,21 @@ public class InProcessPolicyStore implements PolicyStore { .toList(); } + @Override + public void reorder(Long teamId, List orderedIds) { + int position = 0; + for (String id : orderedIds) { + Policy policy = policies.get(id); + if (policy == null || !Objects.equals(policy.teamId(), teamId)) { + continue; + } + sortOrders.put(id, position++); + } + } + @Override public boolean delete(String id) { + sortOrders.remove(id); return policies.remove(id) != null; } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java index 4de6ee2f3a..5089b8d419 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java @@ -1,11 +1,13 @@ package stirling.software.proprietary.policy.store; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import lombok.RequiredArgsConstructor; @@ -26,6 +28,7 @@ public class JpaPolicyStore implements PolicyStore { private final ObjectMapper objectMapper; @Override + @Transactional public Policy save(Policy policy) { String id = policy.id() == null || policy.id().isBlank() @@ -50,11 +53,49 @@ public class JpaPolicyStore implements PolicyStore { entity.setEnabled(stored.enabled()); entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type()); entity.setTeamId(stored.teamId()); + // Preserve an existing policy's run-order position; append a new one to the end of its + // team's queue (max + 1), so setting up a policy adds it last by default. + entity.setSortOrder( + repository + .findById(id) + .map(PolicyEntity::getSortOrder) + .orElseGet(() -> nextSortOrder(stored.teamId()))); entity.setPolicyJson(objectMapper.writeValueAsString(stored)); repository.save(entity); return stored; } + /** + * Append position for a new policy: max(existing) + 1, computed under a pessimistic lock on the + * team's rows (see {@link PolicyRepository#findByTeamForUpdate}) so two concurrent creates + * can't both read the same max and assign a duplicate order. (A brand-new team has no rows to + * lock; a rare simultaneous first-create there ties at 0 β€” harmless, since the ordering query + * breaks ties by id and any later reorder normalises it.) + */ + private int nextSortOrder(Long teamId) { + return repository.findByTeamForUpdate(teamId).stream() + .map(entity -> entity.getSortOrder() == null ? 0 : entity.getSortOrder()) + .max(Integer::compareTo) + .orElse(-1) + + 1; + } + + @Override + @Transactional + public void reorder(Long teamId, List orderedIds) { + int position = 0; + for (String id : orderedIds) { + PolicyEntity entity = repository.findById(id).orElse(null); + // Ignore unknown ids and any policy outside the caller's team β€” a reorder can't reach + // across teams. + if (entity == null || !Objects.equals(entity.getTeamId(), teamId)) { + continue; + } + entity.setSortOrder(position++); + repository.save(entity); + } + } + @Override public Optional get(String id) { return repository.findById(id).map(this::toPolicy); @@ -62,7 +103,7 @@ public class JpaPolicyStore implements PolicyStore { @Override public List all() { - return repository.findAll().stream().map(this::toPolicy).toList(); + return repository.findAllOrdered().stream().map(this::toPolicy).toList(); } @Override diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java index 4b99a90e6c..08e6bd4fbe 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java @@ -47,6 +47,14 @@ public class PolicyEntity implements Serializable { @Column(name = "team_id") private Long teamId; + /** + * Position in the team's run order (ascending). Team-wide and admin-editable; the per-trigger + * order the UI shows is this single sequence filtered by trigger. New policies are appended + * (max + 1). Nullable for pre-existing rows; treated as 0 when sorting. + */ + @Column(name = "sort_order") + private Integer sortOrder; + @Column(name = "policy_json", columnDefinition = "text") private String policyJson; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java index 8cb36b73d8..82ae4355dd 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java @@ -3,10 +3,13 @@ package stirling.software.proprietary.policy.store; import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import jakarta.persistence.LockModeType; + @Repository public interface PolicyRepository extends JpaRepository { @@ -14,12 +17,29 @@ public interface PolicyRepository extends JpaRepository { List findByTriggerTypeAndEnabledTrue(String triggerType); /** - * Policies belonging to a team, loaded without scanning every team's rows. A {@code null} - * teamId matches the rows with no team (login-disabled / pre-team data), mirroring the - * in-memory team filter rather than the empty result a plain {@code = null} would give. + * Policies belonging to a team, in run order (ascending {@code sortOrder}; a null order sorts + * first, id breaks ties for stability). A {@code null} teamId matches the rows with no team + * (login-disabled / pre-team data), mirroring the in-memory team filter rather than the empty + * result a plain {@code = null} would give. */ + @Query( + "select p from PolicyEntity p where ((:teamId is null and p.teamId is null) or" + + " p.teamId = :teamId) order by coalesce(p.sortOrder, 0) asc, p.id asc") + List findByTeam(@Param("teamId") Long teamId); + + /** All policies in run order β€” used when team scoping is off (login-disabled). */ + @Query("select p from PolicyEntity p order by coalesce(p.sortOrder, 0) asc, p.id asc") + List findAllOrdered(); + + /** + * The team's policy rows, locked for the transaction (SELECT … FOR UPDATE). Appending a new + * policy reads the max {@code sortOrder} from these under the lock, so two concurrent creates + * serialize instead of both reading a stale max and assigning the same position. Must be called + * inside a transaction. + */ + @Lock(LockModeType.PESSIMISTIC_WRITE) @Query( "select p from PolicyEntity p where (:teamId is null and p.teamId is null) or" + " p.teamId = :teamId") - List findByTeam(@Param("teamId") Long teamId); + List findByTeamForUpdate(@Param("teamId") Long teamId); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java index 67796d4a67..ac9e210439 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java @@ -21,6 +21,13 @@ public interface PolicyStore { /** Enabled policies with the given trigger type, for background triggers. */ List findByTriggerType(String triggerType); + /** + * Set the team's run order from {@code orderedIds} (position β†’ sortOrder). Only policies that + * belong to {@code teamId} are touched; unknown/other-team ids are ignored, so a caller can't + * reorder across teams. Ids omitted from the list keep their existing order. + */ + void reorder(Long teamId, List orderedIds); + /** Returns whether the policy existed. */ boolean delete(String id); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java index 79ba3e52e1..dee78e8488 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.policy.config.FolderAccessGuard; import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.engine.SweepKind; import stirling.software.proprietary.policy.input.InputSource; import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.Policy; @@ -202,7 +203,8 @@ public class FolderWatchTrigger implements PolicyTrigger { } if (dirs.stream().anyMatch(changedDirs::contains)) { log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name()); - policyRunner.run(policy); + // Light: the periodic reconcile does the full sweep. + policyRunner.run(policy, SweepKind.LIGHT); } } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/InitialSecuritySetup.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/InitialSecuritySetup.java index 94dc41a231..5d68bc99e0 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/InitialSecuritySetup.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/InitialSecuritySetup.java @@ -22,6 +22,7 @@ import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.security.service.DatabaseServiceInterface; import stirling.software.proprietary.security.service.SaveUserRequest; +import stirling.software.proprietary.security.service.TeamMembershipService; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; import stirling.software.proprietary.service.UserLicenseSettingsService; @@ -40,6 +41,7 @@ public class InitialSecuritySetup { private final DatabaseServiceInterface databaseService; private final UserLicenseSettingsService licenseSettingsService; private final Environment environment; + private final TeamMembershipService teamMembershipService; /** * SaaS manages identity in Supabase and billing via PAYG, so the self-host bootstrap steps that @@ -114,6 +116,7 @@ public class InitialSecuritySetup { } userService.saveAll(usersWithoutTeam); // batch save + usersWithoutTeam.forEach(teamMembershipService::syncMembership); if (usersWithoutTeam != null && !usersWithoutTeam.isEmpty()) { log.info( "Assigned {} user(s) without a team to the default team.", diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java index c53d835ca5..2e9c4d3e5b 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java @@ -34,9 +34,11 @@ import stirling.software.common.model.exception.UnsupportedProviderException; "stirling.software.proprietary.workflow.repository", "stirling.software.proprietary.policy.store", "stirling.software.proprietary.policy.source", + "stirling.software.proprietary.policy.ledger", "stirling.software.proprietary.accountlink", "stirling.software.proprietary.access.repository", - "stirling.software.proprietary.integration.repository" + "stirling.software.proprietary.integration.repository", + "stirling.software.proprietary.classification.store" }) @EntityScan({ "stirling.software.proprietary.security.model", @@ -45,9 +47,11 @@ import stirling.software.common.model.exception.UnsupportedProviderException; "stirling.software.proprietary.workflow.model", "stirling.software.proprietary.policy.store", "stirling.software.proprietary.policy.source", + "stirling.software.proprietary.policy.ledger", "stirling.software.proprietary.accountlink", "stirling.software.proprietary.access.model", - "stirling.software.proprietary.integration.model" + "stirling.software.proprietary.integration.model", + "stirling.software.proprietary.classification.store" }) public class DatabaseConfig { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AuthController.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AuthController.java index dcea7807b1..86a1c5fe0c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AuthController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AuthController.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.constants.JwtConstants; import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.access.service.ResourceAccessService; +import stirling.software.proprietary.access.service.TeamLeadLookup; import stirling.software.proprietary.audit.AuditEventType; import stirling.software.proprietary.audit.AuditLevel; import stirling.software.proprietary.audit.Audited; @@ -66,6 +67,7 @@ public class AuthController { private final ApplicationProperties applicationProperties; private final AiUserDataService aiUserDataService; private final ResourceAccessService resourceAccessService; + private final TeamLeadLookup teamLeadLookup; /** * Login endpoint - replaces Supabase signInWithPassword @@ -265,8 +267,11 @@ public class AuthController { .body(Map.of("error", "Not authenticated")); } - UserDetails userDetails = (UserDetails) auth.getPrincipal(); - User user = (User) userDetails; + // Anonymous SaaS sessions carry a raw Jwt principal; treat them as unauthenticated + if (!(auth.getPrincipal() instanceof User user)) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(Map.of("error", "Not authenticated")); + } return ResponseEntity.ok(Map.of("user", buildUserResponse(user))); @@ -631,6 +636,12 @@ public class AuthController { userMap.put("role", user.getRolesAsString()); userMap.put("enabled", user.isEnabled()); userMap.put("portalAccess", resourceAccessService.canAccessPortal(user)); + userMap.put("teamLead", teamLeadLookup.isAnyTeamLeader(user)); + // Expose the caller's team so non-admin team owners can scope their own team's resources. + if (user.getTeam() != null) { + userMap.put( + "team", Map.of("id", user.getTeam().getId(), "name", user.getTeam().getName())); + } userMap.put( "authenticationType", user.getAuthenticationType()); // Expose authentication type for SSO detection diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/TeamController.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/TeamController.java index d9e43fa848..32a429ea72 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/TeamController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/TeamController.java @@ -14,11 +14,15 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.common.annotations.api.TeamApi; +import stirling.software.proprietary.access.model.PrincipalType; +import stirling.software.proprietary.access.repository.ResourceGrantRepository; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.config.PremiumEndpoint; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.security.repository.TeamRepository; +import stirling.software.proprietary.security.service.TeamMembershipService; import stirling.software.proprietary.security.service.TeamService; @TeamApi @@ -29,6 +33,9 @@ public class TeamController { private final TeamRepository teamRepository; private final UserRepository userRepository; + private final ResourceGrantRepository resourceGrantRepository; + private final IntegrationConfigRepository integrationConfigRepository; + private final TeamMembershipService teamMembershipService; @PreAuthorize("hasRole('ADMIN')") @PostMapping("/create") @@ -96,10 +103,72 @@ public class TeamController { "Team must be empty before deletion. Please remove all members first.")); } + if (integrationConfigRepository.existsByOwnerTeam_Id(teamId)) { + return ResponseEntity.status(HttpStatus.CONFLICT) + .body( + Map.of( + "error", + "Team still owns integration configurations. Delete or reassign them first.")); + } + + // Team grants and membership rows would dangle once the team row is gone + resourceGrantRepository.deleteByPrincipalTypeAndPrincipalId(PrincipalType.TEAM, teamId); + teamMembershipService.deleteAllForTeam(teamId); teamRepository.delete(team); return ResponseEntity.ok(Map.of("message", "Team deleted successfully")); } + @PreAuthorize("hasRole('ADMIN')") + @PostMapping("/setOwner") + @Transactional + public ResponseEntity setTeamOwner( + @RequestParam("teamId") Long teamId, @RequestParam("userId") Long userId) { + return mutateOwner(teamId, userId, true); + } + + @PreAuthorize("hasRole('ADMIN')") + @PostMapping("/removeOwner") + @Transactional + public ResponseEntity removeTeamOwner( + @RequestParam("teamId") Long teamId, @RequestParam("userId") Long userId) { + return mutateOwner(teamId, userId, false); + } + + private ResponseEntity mutateOwner(Long teamId, Long userId, boolean owner) { + Optional teamOpt = teamRepository.findById(teamId); + if (teamOpt.isEmpty()) { + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(Map.of("error", "Team not found.")); + } + Team team = teamOpt.get(); + + // System teams have no owners + if (TeamService.INTERNAL_TEAM_NAME.equals(team.getName()) + || TeamService.DEFAULT_TEAM_NAME.equals(team.getName())) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST) + .body(Map.of("error", "System teams cannot have owners.")); + } + + Optional userOpt = userRepository.findById(userId); + if (userOpt.isEmpty()) { + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(Map.of("error", "User not found.")); + } + User user = userOpt.get(); + + if (user.getTeam() == null || !user.getTeam().getId().equals(teamId)) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST) + .body(Map.of("error", "User must be a member of the team.")); + } + + if (owner) { + teamMembershipService.setOwner(team, user); + return ResponseEntity.ok(Map.of("message", "Team owner assigned successfully")); + } + teamMembershipService.removeOwner(team, user); + return ResponseEntity.ok(Map.of("message", "Team owner removed successfully")); + } + @PreAuthorize("hasRole('ADMIN')") @PostMapping("/addUser") @Transactional @@ -138,6 +207,7 @@ public class TeamController { // Assign user to team user.setTeam(team); userRepository.save(user); + teamMembershipService.syncMembership(user); return ResponseEntity.ok(Map.of("message", "User added to team successfully")); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/UserController.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/UserController.java index d8500e475f..b19c052ff1 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/UserController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/UserController.java @@ -50,6 +50,7 @@ import stirling.software.proprietary.security.saml2.CustomSaml2AuthenticatedPrin import stirling.software.proprietary.security.service.EmailService; import stirling.software.proprietary.security.service.LoginAttemptService; import stirling.software.proprietary.security.service.SaveUserRequest; +import stirling.software.proprietary.security.service.TeamMembershipService; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; import stirling.software.proprietary.security.session.SessionPersistentRegistry; @@ -69,6 +70,7 @@ public class UserController { private final Optional emailService; private final UserLicenseSettingsService licenseSettingsService; private final LoginAttemptService loginAttemptService; + private final TeamMembershipService teamMembershipService; @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/register") @@ -644,6 +646,7 @@ public class UserController { user.setTeam(team); userRepository.save(user); + teamMembershipService.syncMembership(user); } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/model/dto/AdminUserSummary.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/model/dto/AdminUserSummary.java index 2a3e7d2af6..5a503e9436 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/model/dto/AdminUserSummary.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/model/dto/AdminUserSummary.java @@ -50,6 +50,14 @@ public class AdminUserSummary { @Schema(description = "Team membership (if any)") private TeamSummary team; + @Schema(description = "Whether the user owns (leads) any team") + private boolean teamLead; + + @Schema( + description = + "Whether the user may access the portal, per the server-side access policy") + private boolean portalAccess; + @Schema(description = "User account creation timestamp") private LocalDateTime createdAt; diff --git a/app/saas/src/main/java/stirling/software/saas/repository/TeamMembershipRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/repository/TeamMembershipRepository.java similarity index 77% rename from app/saas/src/main/java/stirling/software/saas/repository/TeamMembershipRepository.java rename to app/proprietary/src/main/java/stirling/software/proprietary/security/repository/TeamMembershipRepository.java index 36c04f09c3..0662edd67b 100644 --- a/app/saas/src/main/java/stirling/software/saas/repository/TeamMembershipRepository.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/repository/TeamMembershipRepository.java @@ -1,15 +1,17 @@ -package stirling.software.saas.repository; +package stirling.software.proprietary.security.repository; import java.util.List; import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import stirling.software.common.model.enumeration.TeamRole; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.model.TeamMembership; +import stirling.software.proprietary.security.model.User; @Repository public interface TeamMembershipRepository extends JpaRepository { @@ -106,4 +108,24 @@ public interface TeamMembershipRepository extends JpaRepository findByRoleFetchingUserAndTeam(@Param("role") TeamRole role); + + void deleteByTeamId(Long teamId); + + void deleteByUserId(Long userId); + + // Detach invitation references so deleting the inviting user does not hit the FK. + @Modifying + @Query("update TeamMembership tm set tm.invitedBy = null where tm.invitedBy = :user") + void clearInvitedBy(@Param("user") User user); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/TeamMembershipService.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/TeamMembershipService.java new file mode 100644 index 0000000000..e8fca904b6 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/TeamMembershipService.java @@ -0,0 +1,117 @@ +package stirling.software.proprietary.security.service; + +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; + +/** + * Keeps team_memberships in step with users.team_id on self-hosted admin flows and holds the + * team-owner (LEADER role) mutations. SaaS owns membership lifecycle (seats, personal teams, + * last-leader guards) via SaasTeamService, so these mutations no-op on the saas profile to avoid + * corrupting its accounting. + */ +@Service +@RequiredArgsConstructor +public class TeamMembershipService { + + private final TeamMembershipRepository membershipRepository; + private final Environment environment; + + private boolean isSaas() { + return Arrays.asList(environment.getActiveProfiles()).contains("saas"); + } + + /** Reflects users.team_id into membership rows, preserving an existing role on the team. */ + @Transactional + public void syncMembership(User user) { + if (isSaas() || user == null || user.getId() == null) { + return; + } + Long teamId = user.getTeam() != null ? user.getTeam().getId() : null; + boolean present = false; + for (TeamMembership row : membershipRepository.findByUserId(user.getId())) { + if (teamId != null && teamId.equals(row.getTeam().getId())) { + present = true; + } else { + membershipRepository.delete(row); + } + } + if (teamId != null && !present) { + membershipRepository.save(newRow(user.getTeam(), user, TeamRole.MEMBER)); + } + } + + /** Promotes a member to team owner, creating the membership row if it is missing. */ + @Transactional + public void setOwner(Team team, User user) { + if (isSaas()) { + return; + } + Optional existing = + membershipRepository.findByTeamIdAndUserId(team.getId(), user.getId()); + if (existing.isPresent()) { + existing.get().setRole(TeamRole.LEADER); + membershipRepository.save(existing.get()); + } else { + membershipRepository.save(newRow(team, user, TeamRole.LEADER)); + } + } + + /** Demotes a team owner back to member; keeps the membership row. */ + @Transactional + public void removeOwner(Team team, User user) { + if (isSaas()) { + return; + } + membershipRepository + .findByTeamIdAndUserId(team.getId(), user.getId()) + .ifPresent( + row -> { + row.setRole(TeamRole.MEMBER); + membershipRepository.save(row); + }); + } + + /** Owner user ids for a team. */ + @Transactional(readOnly = true) + public List ownerUserIds(Long teamId) { + return membershipRepository.findByTeamIdAndRole(teamId, TeamRole.LEADER).stream() + .map(row -> row.getUser().getId()) + .toList(); + } + + @Transactional + public void deleteAllForTeam(Long teamId) { + membershipRepository.deleteByTeamId(teamId); + } + + @Transactional + public void deleteAllForUser(User user) { + membershipRepository.deleteByUserId(user.getId()); + membershipRepository.clearInvitedBy(user); + } + + private TeamMembership newRow(Team team, User user, TeamRole role) { + TeamMembership row = new TeamMembership(); + row.setTeam(team); + row.setUser(user); + row.setRole(role); + // Self-hosted rows are admin-assigned, not invitation-driven. + row.setInvitedAt(LocalDateTime.now()); + row.setAcceptedAt(LocalDateTime.now()); + return row; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/UserService.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/UserService.java index a7aaa573fd..45532978f3 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/UserService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/UserService.java @@ -39,6 +39,11 @@ import stirling.software.common.model.enumeration.Role; import stirling.software.common.model.exception.UnsupportedProviderException; import stirling.software.common.service.UserServiceInterface; import stirling.software.common.util.RegexPatternUtils; +import stirling.software.proprietary.access.model.PrincipalType; +import stirling.software.proprietary.access.model.ResourceType; +import stirling.software.proprietary.access.repository.ResourceGrantRepository; +import stirling.software.proprietary.integration.model.IntegrationConfig; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.database.repository.AuthorityRepository; import stirling.software.proprietary.security.database.repository.PersistentLoginRepository; @@ -87,6 +92,9 @@ public class UserService implements UserServiceInterface { private final StorageCleanupEntryRepository storageCleanupEntryRepository; private final FileShareRepository fileShareRepository; private final FileShareAccessRepository fileShareAccessRepository; + private final ResourceGrantRepository resourceGrantRepository; + private final IntegrationConfigRepository integrationConfigRepository; + private final TeamMembershipService teamMembershipService; @Transactional public void processSSOPostLogin( @@ -249,6 +257,21 @@ public class UserService implements UserServiceInterface { private void deleteUserRelatedData(User user) { log.info("Deleting all associated data for user: {}", user.getUsername()); + // Drop ACL grants held by this user and detach grants they issued + resourceGrantRepository.deleteByPrincipalTypeAndPrincipalId( + PrincipalType.USER, user.getId()); + resourceGrantRepository.clearGrantedBy(user); + + // Integration configs owned by this user FK the users row; drop them and their grants + for (IntegrationConfig cfg : integrationConfigRepository.findByOwnerUser(user)) { + resourceGrantRepository.deleteByResourceTypeAndResourceId( + ResourceType.INTEGRATION_CONFIG, String.valueOf(cfg.getId())); + } + integrationConfigRepository.deleteByOwnerUser(user); + + // Membership rows and invitation references would dangle once the user row is gone + teamMembershipService.deleteAllForUser(user); + // Delete server certificate (non-nullable OneToOne β†’ User) userServerCertificateService.deleteUserCertificate(user.getId()); @@ -412,6 +435,7 @@ public class UserService implements UserServiceInterface { } user.setTeam(team); userRepository.save(user); + teamMembershipService.syncMembership(user); databaseService.exportDatabase(); } @@ -523,6 +547,7 @@ public class UserService implements UserServiceInterface { // Save user userRepository.save(user); + teamMembershipService.syncMembership(user); // Export database databaseService.exportDatabase(); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index d3b2d21a00..0e9460eca7 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -404,6 +404,7 @@ public class AiWorkflowService { buildCompletedResponse( response.getRationale(), result.files(), + result.origins(), inputFileNames(filesById), result.report())); } catch (InternalApiTimeoutException e) { @@ -465,7 +466,8 @@ public class AiWorkflowService { } }; return new WorkflowState.Terminal( - buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null)); + buildCompletedResponse( + response.getSummary(), List.of(resource), null, List.of(), null)); } @SuppressWarnings("unchecked") @@ -523,7 +525,11 @@ public class AiWorkflowService { return new WorkflowState.Terminal( buildCompletedResponse( - summary, result.files(), inputFileNames(filesById), result.report())); + summary, + result.files(), + result.origins(), + inputFileNames(filesById), + result.report())); } catch (InternalApiTimeoutException e) { log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage()); return new WorkflowState.Terminal( @@ -612,19 +618,35 @@ public class AiWorkflowService { private AiWorkflowResponse buildCompletedResponse( String summary, List resultFiles, + List origins, List inputFileNames, JsonNode report) throws IOException { // Store every output file individually so each gets its own Stirling file ID and the // frontend can add them as independent variants without going through a zip. - boolean preserveInputNames = inputFileNames.size() == resultFiles.size(); + // Count outputs per source so only a clean 1:1 transform (one output for a source) reuses + // the input's name; a split (one input β†’ many outputs) keeps each entry's own name. + Map outputsPerOrigin = + origins == null + ? Map.of() + : origins.stream() + .filter(o -> o != null) + .collect(Collectors.groupingBy(o -> o, Collectors.counting())); List descriptors = new ArrayList<>(); for (int i = 0; i < resultFiles.size(); i++) { Resource resource = resultFiles.get(i); String responseName = resource.getFilename(); - String inputName = preserveInputNames ? inputFileNames.get(i) : null; - // Prefer the input name only for 1:1 operations where the output keeps the same - // extension (rotate, compress, etc.). For converters and other extension-changing + // The output's source input (from the executor), used both to name it and to tell the + // client which file to version in place. + Integer origin = origins != null && i < origins.size() ? origins.get(i) : null; + boolean uniqueOrigin = + origin != null && outputsPerOrigin.getOrDefault(origin, 0L) == 1L; + String inputName = + uniqueOrigin && origin >= 0 && origin < inputFileNames.size() + ? inputFileNames.get(origin) + : null; + // Prefer the source input's name only for 1:1 operations where the output keeps the + // same extension (rotate, compress, etc.). For converters and other extension-changing // tools, the response filename from Content-Disposition is authoritative. String name; if (inputName != null @@ -644,7 +666,11 @@ public class AiWorkflowService { try (java.io.InputStream is = resource.getInputStream()) { fileId = fileStorage.storeInputStream(is, name).fileId(); } - descriptors.add(new AiWorkflowResultFile(fileId, name, contentType)); + // Only expose the source when this is a clean 1:1 transform, so the client can treat a + // present sourceIndex as "replace that input in place" without further disambiguation. + descriptors.add( + new AiWorkflowResultFile( + fileId, name, contentType, uniqueOrigin ? origin : null)); } AiWorkflowResponse completed = new AiWorkflowResponse(); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/security/ResourceAccessSecurityTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/security/ResourceAccessSecurityTest.java new file mode 100644 index 0000000000..09c0fbf1f7 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/security/ResourceAccessSecurityTest.java @@ -0,0 +1,61 @@ +package stirling.software.proprietary.access.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import stirling.software.proprietary.access.service.ResourceAccessService; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.service.UserService; + +@ExtendWith(MockitoExtension.class) +class ResourceAccessSecurityTest { + + @Mock private ResourceAccessService accessService; + @Mock private UserService userService; + + @InjectMocks private ResourceAccessSecurity security; + + @AfterEach + void clearContext() { + SecurityContextHolder.clearContext(); + } + + @Test + void userPrincipalDelegatesToPortalCheck() { + User user = new User(); + user.setId(5L); + authenticate(user); + when(accessService.canAccessPortal(user)).thenReturn(true); + + assertThat(security.canUsePortal()).isTrue(); + } + + @Test + void deniedWithoutAuthentication() { + assertThat(security.canUsePortal()).isFalse(); + } + + @Test + void nonUserPrincipalWithoutBackingRowIsDenied() { + // Mirrors an anonymous SaaS session: principal is not a User and resolves to nothing. + authenticate("anonymousUser"); + + assertThat(security.canUsePortal()).isFalse(); + } + + private void authenticate(Object principal) { + SecurityContextHolder.getContext() + .setAuthentication( + new UsernamePasswordAuthenticationToken( + principal, null, java.util.List.of())); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/DefaultPrincipalResolverTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/DefaultPrincipalResolverTest.java new file mode 100644 index 0000000000..94e99bdbc9 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/DefaultPrincipalResolverTest.java @@ -0,0 +1,54 @@ +package stirling.software.proprietary.access.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.access.model.PrincipalRef; +import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.security.model.User; + +class DefaultPrincipalResolverTest { + + private final DefaultPrincipalResolver resolver = new DefaultPrincipalResolver(); + + @Test + void userWithoutTeamProjectsUser() { + assertThat(resolver.principalsOf(user(5, null))) + .containsExactlyInAnyOrder(PrincipalRef.user(5L)); + } + + @Test + void userWithTeamProjectsUserAndTeam() { + assertThat(resolver.principalsOf(user(5, 7L))) + .containsExactlyInAnyOrder(PrincipalRef.user(5L), PrincipalRef.team(7L)); + } + + @Test + void nullUserProjectsNothing() { + assertThat(resolver.principalsOf(null)).isEmpty(); + assertThat(resolver.principalTokens(null)).isEmpty(); + } + + @Test + void tokensUseTheCanonicalWireForm() { + assertThat(resolver.principalTokens(user(5, 7L))) + .containsExactlyInAnyOrder("user:5", "team:7"); + } + + @Test + void selfHostedAllowsDeploymentWideAccess() { + assertThat(resolver.allowsDeploymentWideAccess()).isTrue(); + } + + private User user(long id, Long teamId) { + User u = new User(); + u.setId(id); + if (teamId != null) { + Team t = new Team(); + t.setId(teamId); + u.setTeam(t); + } + return u; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/MembershipTeamLeadLookupTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/MembershipTeamLeadLookupTest.java new file mode 100644 index 0000000000..f485bcec52 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/MembershipTeamLeadLookupTest.java @@ -0,0 +1,55 @@ +package stirling.software.proprietary.access.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; + +@ExtendWith(MockitoExtension.class) +class MembershipTeamLeadLookupTest { + + @Mock private TeamMembershipRepository memberships; + + @InjectMocks private MembershipTeamLeadLookup lookup; + + @Test + void leaderMembershipMakesTeamLeader() { + when(memberships.existsByTeamIdAndUserIdAndRole(7L, 5L, TeamRole.LEADER)).thenReturn(true); + assertThat(lookup.isLeaderOfTeam(user(5), 7L)).isTrue(); + } + + @Test + void memberOnlyIsNotTeamLeader() { + when(memberships.existsByTeamIdAndUserIdAndRole(7L, 5L, TeamRole.LEADER)).thenReturn(false); + assertThat(lookup.isLeaderOfTeam(user(5), 7L)).isFalse(); + } + + @Test + void anyLeadershipDetectedAcrossTeams() { + when(memberships.existsByUserIdAndRole(5L, TeamRole.LEADER)).thenReturn(true); + assertThat(lookup.isAnyTeamLeader(user(5))).isTrue(); + } + + @Test + void nullInputsNeverQueryAndDeny() { + assertThat(lookup.isAnyTeamLeader(null)).isFalse(); + assertThat(lookup.isLeaderOfTeam(null, 7L)).isFalse(); + assertThat(lookup.isLeaderOfTeam(user(5), null)).isFalse(); + verifyNoInteractions(memberships); + } + + private User user(long id) { + User u = new User(); + u.setId(id); + return u; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/OwnershipServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/OwnershipServiceTest.java index 3f70058d0b..f2d86e783d 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/OwnershipServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/OwnershipServiceTest.java @@ -3,6 +3,7 @@ package stirling.software.proprietary.access.service; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.Optional; @@ -150,6 +151,27 @@ class OwnershipServiceTest { assertThat(ownership.canUse(TYPE, r, user(8))).isFalse(); // not a lead } + @Test + void disabledResourceManageableOnlyByOwnerOrAdmin() { + TestResource r = new TestResource(1L); + r.setEnabled(false); + r.setOwnerUser(user(7)); + + assertThat(ownership.canManage(TYPE, r, user(7))).isTrue(); // owner + assertThat(ownership.canManage(TYPE, r, user(8))).isFalse(); // would-be MANAGE grantee + assertThat(ownership.canManage(TYPE, r, admin(2))).isTrue(); // admin + // Grants are bypassed entirely while disabled. + verifyNoInteractions(accessService); + } + + @Test + void enabledResourceManageDelegatesToTheAcl() { + TestResource r = new TestResource(1L); + when(accessService.canManageResource(any(), any(), any(), any())).thenReturn(true); + + assertThat(ownership.canManage(TYPE, r, user(7))).isTrue(); + } + // ---- helpers ---- private void assertForbidden(org.assertj.core.api.ThrowableAssert.ThrowingCallable call) { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java index 230c207eb0..baa6755493 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/ResourceAccessServiceTest.java @@ -1,20 +1,22 @@ package stirling.software.proprietary.access.service; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import java.lang.reflect.Field; import java.util.List; +import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import stirling.software.proprietary.access.model.AccessPermission; import stirling.software.proprietary.access.model.DefaultAccessPolicy; +import stirling.software.proprietary.access.model.PrincipalRef; import stirling.software.proprietary.access.model.PrincipalType; import stirling.software.proprietary.access.model.ResourceGrant; import stirling.software.proprietary.access.model.ResourceType; @@ -32,13 +34,20 @@ class ResourceAccessServiceTest { @Mock private ResourceGrantRepository grantRepository; @Mock private TeamLeadLookup teamLeadLookup; - @InjectMocks private ResourceAccessService service; + private ResourceAccessService service; @BeforeEach - void setPortalDefault() throws Exception { + void setUp() throws Exception { + service = newService(new DefaultPrincipalResolver()); + } + + private ResourceAccessService newService(PrincipalResolver resolver) throws Exception { + ResourceAccessService s = + new ResourceAccessService(grantRepository, teamLeadLookup, resolver); Field f = ResourceAccessService.class.getDeclaredField("portalDefaultPolicy"); f.setAccessible(true); - f.set(service, DefaultAccessPolicy.ADMINS_AND_TEAM_LEADS); + f.set(s, DefaultAccessPolicy.ADMINS_AND_TEAM_LEADS); + return s; } // ---- owner / admin short-circuits ---- @@ -55,15 +64,56 @@ class ResourceAccessServiceTest { void ownerMayUseEvenWithExplicitOnly() { assertThat( service.canUseResource( - TYPE, RID, 5L, DefaultAccessPolicy.EXPLICIT_ONLY, user(5))) + TYPE, + RID, + PrincipalRef.user(5L), + DefaultAccessPolicy.EXPLICIT_ONLY, + user(5))) .isTrue(); } @Test void nullUserIsAlwaysDenied() { - assertThat(service.canUseResource(TYPE, RID, 5L, DefaultAccessPolicy.ORG_ALL, null)) + assertThat( + service.canUseResource( + TYPE, + RID, + PrincipalRef.user(5L), + DefaultAccessPolicy.ORG_ALL, + null)) + .isFalse(); + assertThat(service.canManageResource(TYPE, RID, PrincipalRef.user(5L), null)).isFalse(); + } + + // ---- owner refs ---- + + @Test + void teamOwnerRefAllowsLeaderOfThatTeam() { + User leader = userInTeam(5, 7); + when(teamLeadLookup.isLeaderOfTeam(leader, 7L)).thenReturn(true); + assertThat( + service.canUseResource( + TYPE, + RID, + PrincipalRef.team(7L), + DefaultAccessPolicy.EXPLICIT_ONLY, + leader)) + .isTrue(); + assertThat(service.canManageResource(TYPE, RID, PrincipalRef.team(7L), leader)).isTrue(); + } + + @Test + void teamOwnerRefDeniesPlainTeamMember() { + stubGrants(); + User member = userInTeam(5, 7); + assertThat( + service.canUseResource( + TYPE, + RID, + PrincipalRef.team(7L), + DefaultAccessPolicy.EXPLICIT_ONLY, + member)) .isFalse(); - assertThat(service.canManageResource(TYPE, RID, 5L, null)).isFalse(); } // ---- explicit grants ---- @@ -124,15 +174,60 @@ class ResourceAccessServiceTest { assertThat(service.canManageResource(TYPE, RID, null, user(5))).isTrue(); } + // ---- upsert semantics ---- + + @Test + void grantOverwritesPermissionEvenDowngrading() { + // Upsert key is (type, resourceId, principalType, principalId); permission is overwritten. + ResourceGrant existing = grant(PrincipalType.USER, 5L, AccessPermission.MANAGE); + stubGrants(existing); + when(grantRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + + ResourceGrant saved = + service.grant(TYPE, RID, PrincipalType.USER, 5L, AccessPermission.USE, null); + + assertThat(saved).isSameAs(existing); + assertThat(saved.getPermission()).isEqualTo(AccessPermission.USE); + } + + // ---- granted resource ids ---- + + @Test + void grantedResourceIdsCollectsAcrossAllPrincipals() { + when(grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId( + TYPE, PrincipalType.USER, 5L)) + .thenReturn(List.of(grantOn("a"))); + when(grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId( + TYPE, PrincipalType.TEAM, 7L)) + .thenReturn(List.of(grantOn("b"))); + + assertThat(service.grantedResourceIds(TYPE, userInTeam(5, 7))) + .containsExactlyInAnyOrder("a", "b"); + } + // ---- default policies ---- @Test - void orgAllDefaultAllowsAnyUser() { + void orgAllDefaultAllowsAnyUserWhenDeploymentWide() { + // Self-hosted resolver allows deployment-wide access, so ORG_ALL admits anyone. stubGrants(); assertThat(service.canUseResource(TYPE, RID, null, DefaultAccessPolicy.ORG_ALL, user(5))) .isTrue(); } + @Test + void orgAllDefaultDeniedWhenNotDeploymentWide() throws Exception { + // Mirrors the saas resolver (USER/TEAM only, no deployment-wide access): ORG_ALL must not + // leak a resource to a tenant's users, so an ungranted user is denied. + ResourceAccessService tenantScoped = + newService(u -> u == null ? Set.of() : Set.of(PrincipalRef.user(u.getId()))); + stubGrants(); + assertThat( + tenantScoped.canUseResource( + TYPE, RID, null, DefaultAccessPolicy.ORG_ALL, user(5))) + .isFalse(); + } + @Test void explicitOnlyDefaultDeniesUngrantedUser() { stubGrants(); @@ -194,6 +289,12 @@ class ResourceAccessServiceTest { return g; } + private ResourceGrant grantOn(String resourceId) { + ResourceGrant g = grant(PrincipalType.USER, 5L, AccessPermission.USE); + g.setResourceId(resourceId); + return g; + } + private User user(long id) { User u = new User(); u.setId(id); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/SecretMaskerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/SecretMaskerTest.java index ea5597cb12..4f76b06b5c 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/access/service/SecretMaskerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/access/service/SecretMaskerTest.java @@ -49,6 +49,23 @@ class SecretMaskerTest { assertThat(merged.get("secretKey")).isEqualTo("NEW"); } + @Test + void mergeDropsKeysAbsentFromIncoming() { + // PUT/replace semantics: a key removed in the edit is removed from storage. + Map stored = new java.util.LinkedHashMap<>(); + stored.put("bucket", "b"); + stored.put("endpoint", "https://old"); + stored.put("secretKey", "REAL"); + Map incoming = new java.util.LinkedHashMap<>(); + incoming.put("bucket", "b"); + incoming.put("secretKey", SecretMasker.MASK); + + Map merged = masker.merge(stored, incoming); + + assertThat(merged).doesNotContainKey("endpoint"); + assertThat(merged.get("secretKey")).isEqualTo("REAL"); // masked secret retained + } + @Test void sanitizeDropsBlankSecretsOnCreate() { Map incoming = new LinkedHashMap<>(); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/classification/ClassificationLabelsControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/classification/ClassificationLabelsControllerTest.java new file mode 100644 index 0000000000..221d567c31 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/classification/ClassificationLabelsControllerTest.java @@ -0,0 +1,133 @@ +package stirling.software.proprietary.classification; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.server.ResponseStatusException; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.UserServiceInterface; +import stirling.software.proprietary.classification.model.ClassificationLabel; +import stirling.software.proprietary.classification.model.ClassificationLabels; +import stirling.software.proprietary.classification.store.ClassificationLabelStore; +import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; + +@ExtendWith(MockitoExtension.class) +@DisplayName("ClassificationLabelsController") +class ClassificationLabelsControllerTest { + + private static final Long TEAM = 7L; + + @Mock private PolicyManagementAuthority policyManagementAuthority; + @Mock private UserServiceInterface userService; + + private ClassificationLabelStore store; + private ApplicationProperties applicationProperties; + private ClassificationLabelsController controller; + + @BeforeEach + void setUp() { + store = new InProcessClassificationLabelStore(); + applicationProperties = new ApplicationProperties(); + controller = + new ClassificationLabelsController( + store, policyManagementAuthority, applicationProperties, userService); + } + + private static ClassificationLabels sample() { + return new ClassificationLabels( + List.of( + new ClassificationLabel("invoice", "Invoice", "receipt-long"), + new ClassificationLabel("contract", "Contract", null))); + } + + private void loginEnabled(boolean enabled) { + applicationProperties.getSecurity().setEnableLogin(enabled); + } + + @Test + @DisplayName("GET returns 204 when the team has no labels") + void getEmpty() { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + ResponseEntity response = controller.getTeamLabels(); + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + } + + @Test + @DisplayName("PUT then GET round-trips the team's labels (login disabled)") + void saveThenGet() { + loginEnabled(false); + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + + controller.saveTeamLabels(sample()); + ResponseEntity got = controller.getTeamLabels(); + + assertThat(got.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(got.getBody()).isNotNull(); + assertThat(got.getBody().labels()).hasSize(2); + assertThat(got.getBody().labels().getFirst().name()).isEqualTo("Invoice"); + assertThat(got.getBody().labels().getFirst().icon()).isEqualTo("receipt-long"); + } + + @Test + @DisplayName("PUT is scoped per team") + void perTeam() { + loginEnabled(false); + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + controller.saveTeamLabels(sample()); + + when(policyManagementAuthority.currentUserTeamId()).thenReturn(99L); + assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + } + + @Test + @DisplayName("PUT is rejected for a non-editor when login is enabled") + void putForbiddenForNonEditor() { + loginEnabled(true); + when(policyManagementAuthority.canEditPolicies()).thenReturn(false); + + assertThatThrownBy(() -> controller.saveTeamLabels(sample())) + .isInstanceOf(ResponseStatusException.class) + .hasFieldOrPropertyWithValue("statusCode", HttpStatus.FORBIDDEN); + } + + @Test + @DisplayName("PUT rejects an invalid label set with 400") + void putInvalid() { + loginEnabled(false); + ClassificationLabels duplicate = + new ClassificationLabels( + List.of( + new ClassificationLabel("invoice", "Invoice", null), + new ClassificationLabel("invoice", "Invoice", null))); + + assertThatThrownBy(() -> controller.saveTeamLabels(duplicate)) + .isInstanceOf(ResponseStatusException.class) + .hasFieldOrPropertyWithValue("statusCode", HttpStatus.BAD_REQUEST); + } + + @Test + @DisplayName("DELETE resets the team back to no stored labels") + void deleteResets() { + loginEnabled(false); + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + controller.saveTeamLabels(sample()); + + ResponseEntity response = controller.resetTeamLabels(); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/classification/model/LabelsValidatorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/classification/model/LabelsValidatorTest.java new file mode 100644 index 0000000000..b9730370c7 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/classification/model/LabelsValidatorTest.java @@ -0,0 +1,146 @@ +package stirling.software.proprietary.classification.model; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import java.util.Locale; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("LabelsValidator") +class LabelsValidatorTest { + + private static ClassificationLabels labels(ClassificationLabel... labels) { + return new ClassificationLabels(List.of(labels)); + } + + private static ClassificationLabel label(String name) { + return new ClassificationLabel(slug(name), name, null); + } + + private static String slug(String name) { + return name.trim() + .toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9]+", "-") + .replaceAll("(^-|-$)", ""); + } + + @Test + @DisplayName("accepts a well-formed label set") + void acceptsValid() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice", "Invoice", "receipt-long"), + label("Contract")); + assertThatCode(() -> LabelsValidator.validate(set)).doesNotThrowAnyException(); + } + + @Test + @DisplayName("accepts an empty label set (reads as: use the default)") + void acceptsEmpty() { + assertThatCode(() -> LabelsValidator.validate(labels())).doesNotThrowAnyException(); + } + + @Test + @DisplayName("rejects a null label set") + void rejectsNull() { + assertThatThrownBy(() -> LabelsValidator.validate(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Labels are required"); + } + + @Test + @DisplayName("rejects duplicate names (distinct ids)") + void rejectsDuplicateNames() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice-a", "Invoice", null), + new ClassificationLabel("invoice-b", "Invoice", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Duplicate label name"); + } + + @Test + @DisplayName("rejects duplicate names differing only by case") + void rejectsDuplicateNamesCaseInsensitive() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice-a", "Invoice", null), + new ClassificationLabel("invoice-b", "INVOICE", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Duplicate label name"); + } + + @Test + @DisplayName("rejects duplicate ids") + void rejectsDuplicateIds() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice", "Invoice", null), + new ClassificationLabel("invoice", "Sales invoice", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Duplicate label id"); + } + + @Test + @DisplayName("rejects a blank name") + void rejectsBlankName() { + ClassificationLabels set = labels(new ClassificationLabel("blank", " ", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Label name must not be blank"); + } + + @Test + @DisplayName("rejects a blank id") + void rejectsBlankId() { + ClassificationLabels set = labels(new ClassificationLabel(" ", "Invoice", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Label id must not be blank"); + } + + @Test + @DisplayName("rejects an over-long name") + void rejectsOverLongName() { + ClassificationLabels set = + labels( + new ClassificationLabel( + "x", "x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1), null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("too long"); + } + + @Test + @DisplayName("rejects an over-long icon (a null icon is fine)") + void rejectsOverLongIcon() { + ClassificationLabels set = + labels( + new ClassificationLabel( + "invoice", + "Invoice", + "x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1))); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("icon is too long"); + } + + @Test + @DisplayName("rejects more labels than the cap") + void rejectsTooManyLabels() { + List tooMany = + IntStream.rangeClosed(0, LabelsValidator.MAX_LABELS) + .mapToObj(i -> label("label" + i)) + .toList(); + assertThatThrownBy(() -> LabelsValidator.validate(new ClassificationLabels(tooMany))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Too many labels"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java new file mode 100644 index 0000000000..c5ee39be14 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java @@ -0,0 +1,176 @@ +package stirling.software.proprietary.controller.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.web.multipart.MultipartFile; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.classification.model.ClassificationLabel; +import stirling.software.proprietary.classification.model.ClassificationLabels; +import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; +import stirling.software.proprietary.service.AiEngineClient; +import stirling.software.proprietary.service.PdfContentExtractor; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ClassifyLabelControllerTest { + + private static final Long TEAM = 7L; + + @Mock private CustomPDFDocumentFactory pdfDocumentFactory; + @Mock private TempFileManager tempFileManager; + @Mock private PdfContentExtractor pdfContentExtractor; + @Mock private PdfMetadataService pdfMetadataService; + @Mock private AiEngineClient aiEngineClient; + @Mock private PolicyManagementAuthority policyManagementAuthority; + + private final ObjectMapper objectMapper = JsonMapper.builder().build(); + private InProcessClassificationLabelStore labelStore; + private ClassifyLabelController controller; + + @BeforeEach + void setUp() { + labelStore = new InProcessClassificationLabelStore(); + controller = + new ClassifyLabelController( + pdfDocumentFactory, + tempFileManager, + pdfContentExtractor, + pdfMetadataService, + aiEngineClient, + objectMapper, + null, + labelStore, + policyManagementAuthority); + } + + private void stubSinglePageDocument() throws Exception { + PDDocument document = mock(PDDocument.class); + MultipartFile file = mock(MultipartFile.class); + when(file.getOriginalFilename()).thenReturn("invoice.pdf"); + when(pdfDocumentFactory.load(any(MultipartFile.class), eq(true))).thenReturn(document); + when(document.getNumberOfPages()).thenReturn(1); + when(pdfContentExtractor.extractPageTextRaw(document, 1)) + .thenReturn("Invoice total due 100.00"); + when(aiEngineClient.post(eq("/api/v1/documents/classify"), anyString(), isNull())) + .thenReturn("{\"outcome\":\"classification\",\"labels\":[\"invoice\"]}"); + + try { + controller.classifyAndLabel(file); + } catch (Exception ignored) { + // WebResponseUtils.pdfDocToWebResponse needs a real temp file; the engine call and + // metadata write we assert on have already happened by the time it runs. + } + } + + private JsonNode sentEngineRequest() throws Exception { + ArgumentCaptor body = ArgumentCaptor.forClass(String.class); + verify(aiEngineClient).post(eq("/api/v1/documents/classify"), body.capture(), isNull()); + return objectMapper.readTree(body.getValue()); + } + + @Test + void classifyAndLabel_writesClassificationWithoutOutcome() throws Exception { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + labelStore.save( + TEAM, + new ClassificationLabels( + List.of(new ClassificationLabel("invoice", "Invoice", null))), + "admin"); + + stubSinglePageDocument(); + + ArgumentCaptor value = ArgumentCaptor.forClass(String.class); + verify(pdfMetadataService) + .setClassificationMetadata(any(PDDocument.class), value.capture()); + + JsonNode written = objectMapper.readTree(value.getValue()); + assertThat(written.has("outcome")).isFalse(); + // The engine returns label ids; they're stored on the document verbatim. + assertThat(written.get("labels").get(0).asText()).isEqualTo("invoice"); + } + + @Test + void classifyAndLabel_sendsTeamLabelIdsAndNames() throws Exception { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + labelStore.save( + TEAM, + new ClassificationLabels( + List.of( + new ClassificationLabel("invoice", "Invoice", "receipt-long"), + new ClassificationLabel("contract", "Contract", null), + new ClassificationLabel("timesheet", "Timesheet", null))), + "admin"); + + stubSinglePageDocument(); + + JsonNode request = sentEngineRequest(); + assertThat(request.get("fileName").asText()).isEqualTo("invoice.pdf"); + JsonNode labels = request.get("labels"); + assertThat(labels.isArray()).isTrue(); + assertThat(labels.size()).isEqualTo(3); + // Each entry is an {id, name} pair β€” the model reasons over names, we store ids. + assertThat( + List.of( + labels.get(0).get("id").asText(), + labels.get(1).get("id").asText(), + labels.get(2).get("id").asText())) + .containsExactly("invoice", "contract", "timesheet"); + assertThat( + List.of( + labels.get(0).get("name").asText(), + labels.get(1).get("name").asText(), + labels.get(2).get("name").asText())) + .containsExactly("Invoice", "Contract", "Timesheet"); + } + + @Test + void classifyAndLabel_skipsClassificationWhenNothingStored() throws Exception { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + + stubSinglePageDocument(); + + // No team labels stored, and the engine holds no default of its own, so the file is passed + // through unlabelled: neither the engine nor the metadata write is invoked. + verify(aiEngineClient, never()).post(anyString(), anyString(), any()); + verify(pdfMetadataService, never()) + .setClassificationMetadata(any(PDDocument.class), anyString()); + } + + @Test + void windowPageNumbers_takesFirstAndLastWithoutOverlap() { + assertEquals(List.of(1, 2, 4, 5), ClassifyLabelController.windowPageNumbers(5, 2)); + assertEquals(List.of(1, 2, 3), ClassifyLabelController.windowPageNumbers(3, 2)); + // Short docs clamp + dedupe rather than throwing or going out of range. + assertEquals(List.of(1, 2), ClassifyLabelController.windowPageNumbers(2, 2)); + assertEquals(List.of(1), ClassifyLabelController.windowPageNumbers(1, 2)); + assertEquals(List.of(), ClassifyLabelController.windowPageNumbers(0, 2)); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerMoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerMoreTest.java index 3ddd9e9a1e..585edc6e66 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerMoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerMoreTest.java @@ -24,6 +24,7 @@ import org.springframework.security.oauth2.core.user.OAuth2User; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.enumeration.Role; +import stirling.software.proprietary.access.service.ResourceAccessService; import stirling.software.proprietary.config.AuditConfigurationProperties; import stirling.software.proprietary.controller.api.ProprietaryUIDataController.AccountData; import stirling.software.proprietary.controller.api.ProprietaryUIDataController.AdminSettingsData; @@ -39,6 +40,7 @@ import stirling.software.proprietary.security.database.repository.SessionReposit import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.Authority; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.saml2.CustomSaml2AuthenticatedPrincipal; import stirling.software.proprietary.security.service.DatabaseServiceInterface; @@ -57,12 +59,14 @@ class ProprietaryUIDataControllerMoreTest { @Mock private SessionPersistentRegistry sessionPersistentRegistry; @Mock private UserRepository userRepository; @Mock private TeamRepository teamRepository; + @Mock private TeamMembershipRepository teamMembershipRepository; @Mock private SessionRepository sessionRepository; @Mock private DatabaseServiceInterface databaseService; @Mock private UserLicenseSettingsService licenseSettingsService; @Mock private PersistentAuditEventRepository auditRepository; @Mock private MfaService mfaService; @Mock private LoginAttemptService loginAttemptService; + @Mock private ResourceAccessService resourceAccessService; private ApplicationProperties applicationProperties; private AuditConfigurationProperties auditConfig; @@ -87,6 +91,7 @@ class ProprietaryUIDataControllerMoreTest { sessionPersistentRegistry, userRepository, teamRepository, + teamMembershipRepository, sessionRepository, databaseService, objectMapper, @@ -94,7 +99,8 @@ class ProprietaryUIDataControllerMoreTest { licenseSettingsService, auditRepository, mfaService, - loginAttemptService); + loginAttemptService, + resourceAccessService); } private static User normalUser(Long id, String username) { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerTest.java index b8552e274c..0b01c9bcf5 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ProprietaryUIDataControllerTest.java @@ -18,6 +18,7 @@ import org.springframework.security.authentication.UsernamePasswordAuthenticatio import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.enumeration.Role; +import stirling.software.proprietary.access.service.ResourceAccessService; import stirling.software.proprietary.config.AuditConfigurationProperties; import stirling.software.proprietary.controller.api.ProprietaryUIDataController.AccountData; import stirling.software.proprietary.controller.api.ProprietaryUIDataController.DatabaseData; @@ -27,6 +28,7 @@ import stirling.software.proprietary.security.database.repository.SessionReposit import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.Authority; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.service.DatabaseService; import stirling.software.proprietary.security.service.LoginAttemptService; @@ -43,12 +45,14 @@ class ProprietaryUIDataControllerTest { @Mock private SessionPersistentRegistry sessionPersistentRegistry; @Mock private UserRepository userRepository; @Mock private TeamRepository teamRepository; + @Mock private TeamMembershipRepository teamMembershipRepository; @Mock private SessionRepository sessionRepository; @Mock private DatabaseService databaseService; @Mock private UserLicenseSettingsService licenseSettingsService; @Mock private PersistentAuditEventRepository auditRepository; @Mock private MfaService mfaService; @Mock private LoginAttemptService loginAttemptService; + @Mock private ResourceAccessService resourceAccessService; private ApplicationProperties applicationProperties; private AuditConfigurationProperties auditConfig; @@ -75,6 +79,7 @@ class ProprietaryUIDataControllerTest { sessionPersistentRegistry, userRepository, teamRepository, + teamMembershipRepository, sessionRepository, databaseService, objectMapper, @@ -82,7 +87,8 @@ class ProprietaryUIDataControllerTest { licenseSettingsService, auditRepository, mfaService, - loginAttemptService); + loginAttemptService, + resourceAccessService); } @Test diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/integration/service/IntegrationConfigServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/integration/service/IntegrationConfigServiceTest.java index 9f3b2d84af..8e0ef200bb 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/integration/service/IntegrationConfigServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/integration/service/IntegrationConfigServiceTest.java @@ -49,6 +49,9 @@ class IntegrationConfigServiceTest { @Mock private OwnershipService ownership; @Mock private SecretMasker secretMasker; + @Mock + private stirling.software.proprietary.access.repository.ResourceGrantRepository grantRepository; + @InjectMocks private IntegrationConfigService service; @Test @@ -58,9 +61,9 @@ class IntegrationConfigServiceTest { User user = user(7); IntegrationConfig created = - service.create(request(IntegrationType.S3, OwnerScope.USER, null), user); + service.create(request(IntegrationType.API, OwnerScope.USER, null), user); - assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.S3); + assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.API); assertThat(created.getName()).isEqualTo("name"); verify(ownership) .assignOwnership(eq(created), eq(OwnerScope.USER), isNull(), eq(user), any()); @@ -156,6 +159,60 @@ class IntegrationConfigServiceTest { .isEqualTo(HttpStatus.FORBIDDEN); } + // ---- S3 type policy ---- + + @Test + void s3PersonalCreateForbiddenForRegularUser() { + User user = user(7); + when(ownership.isAdmin(user)).thenReturn(false); + + assertThatThrownBy( + () -> + service.create( + request(IntegrationType.S3, OwnerScope.USER, null), user)) + .isInstanceOf(ResponseStatusException.class) + .extracting(e -> ((ResponseStatusException) e).getStatusCode()) + .isEqualTo(HttpStatus.FORBIDDEN); + } + + @Test + void s3PersonalCreateAllowedForAdmin() { + when(secretMasker.sanitize(any())).thenReturn(Map.of("bucket", "b")); + when(repository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + User admin = user(1); + when(ownership.isAdmin(admin)).thenReturn(true); + + IntegrationConfig created = + service.create(request(IntegrationType.S3, OwnerScope.USER, null), admin); + + assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.S3); + } + + @Test + void s3TeamScopeCreateDelegatesLeadershipToOwnership() { + // TEAM scope skips the personal-S3 gate; assignOwnership enforces admin/team-owner. + when(secretMasker.sanitize(any())).thenReturn(Map.of("bucket", "b")); + when(repository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + User leader = user(7); + + IntegrationConfig created = + service.create(request(IntegrationType.S3, OwnerScope.TEAM, 3L), leader); + + verify(ownership) + .assignOwnership(eq(created), eq(OwnerScope.TEAM), eq(3L), eq(leader), any()); + } + + @Test + void mcpPersonalCreateAllowedForRegularUser() { + when(secretMasker.sanitize(any())).thenReturn(Map.of("token", "t")); + when(repository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + + IntegrationConfig created = + service.create(request(IntegrationType.MCP, OwnerScope.USER, null), user(7)); + + assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.MCP); + } + // ---- helpers ---- private IntegrationConfig config(long id) { diff --git a/app/saas/src/test/java/stirling/software/saas/model/TeamMembershipTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/model/TeamMembershipTest.java similarity index 98% rename from app/saas/src/test/java/stirling/software/saas/model/TeamMembershipTest.java rename to app/proprietary/src/test/java/stirling/software/proprietary/model/TeamMembershipTest.java index a94e8b9373..653fcd0ef1 100644 --- a/app/saas/src/test/java/stirling/software/saas/model/TeamMembershipTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/model/TeamMembershipTest.java @@ -1,4 +1,4 @@ -package stirling.software.saas.model; +package stirling.software.proprietary.model; import static org.assertj.core.api.Assertions.assertThat; @@ -9,7 +9,6 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import stirling.software.common.model.enumeration.TeamRole; -import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.model.User; /** Constructor, accessor, equals/hashCode/toString, and role-helper tests for TeamMembership. */ diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java index 8a9a085653..f38263ad3e 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java @@ -34,6 +34,7 @@ import stirling.software.proprietary.policy.engine.PolicyRunHandle; import stirling.software.proprietary.policy.engine.PolicyRunRegistry; import stirling.software.proprietary.policy.engine.PolicyRunner; import stirling.software.proprietary.policy.engine.PolicyValidator; +import stirling.software.proprietary.policy.ledger.ProcessedLedger; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.PipelineStep; import stirling.software.proprietary.policy.model.Policy; @@ -62,6 +63,8 @@ class PolicyControllerTest { private stirling.software.proprietary.policy.overview.PolicyOverviewService policyOverviewService; + @Mock private ProcessedLedger processedLedger; + @Mock private TempFileManager tempFileManager; @Mock private JobOwnershipService jobOwnershipService; @@ -89,6 +92,7 @@ class PolicyControllerTest { policyManagementAuthority, policyTriggerManager, policyOverviewService, + processedLedger, policyTriggers, applicationProperties, tempFileManager, @@ -398,6 +402,7 @@ class PolicyControllerTest { ResponseEntity response = controller.deletePolicy("a"); assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + verify(processedLedger).clearPolicy("a"); verify(policyTriggerManager).notifyPoliciesChanged(); } @@ -431,6 +436,53 @@ class PolicyControllerTest { } } + @Nested + @DisplayName("clearProcessedHistory") + class ClearProcessedHistory { + + @Test + @DisplayName("clears an accessible policy's history") + void clears() { + applicationProperties.getSecurity().setEnableLogin(false); + Policy p = policy("a", 1L); + when(policyStore.get("a")).thenReturn(Optional.of(p)); + when(policyAccessGuard.canAccess(p)).thenReturn(true); + + ResponseEntity response = controller.clearProcessedHistory("a"); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + verify(processedLedger).clearPolicy("a"); + } + + @Test + @DisplayName("returns 404 when policy is not accessible") + void notAccessible() { + applicationProperties.getSecurity().setEnableLogin(false); + Policy p = policy("a", 1L); + when(policyStore.get("a")).thenReturn(Optional.of(p)); + when(policyAccessGuard.canAccess(p)).thenReturn(false); + + ResponseEntity response = controller.clearProcessedHistory("a"); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND); + verify(processedLedger, never()).clearPolicy(any()); + } + + @Test + @DisplayName("forbidden when login enabled and caller cannot edit") + void forbidden() { + applicationProperties.getSecurity().setEnableLogin(true); + when(policyManagementAuthority.canEditPolicies()).thenReturn(false); + + assertThatThrownBy(() -> controller.clearProcessedHistory("a")) + .isInstanceOf(ResponseStatusException.class) + .satisfies( + e -> + assertThat(((ResponseStatusException) e).getStatusCode()) + .isEqualTo(HttpStatus.FORBIDDEN)); + } + } + @Nested @DisplayName("runStoredPolicy") class RunStoredPolicy { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java index b1ab05f8ba..13189b2481 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java @@ -4,15 +4,19 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; @@ -24,7 +28,9 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.input.ResolveContext; import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.ledger.ProcessedLedger; import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.OutputSpec; import stirling.software.proprietary.policy.model.PipelineStep; @@ -39,15 +45,15 @@ import stirling.software.proprietary.policy.source.Source; import stirling.software.proprietary.policy.source.SourceStore; /** - * Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs. Verifies - * it pulls every source, runs one job per unit of work, feeds each unit's completion hook the run - * outcome, and that a generator (no sources) still runs once. + * Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs, and the + * orchestrator of ledger hygiene (presence stamping + cleanup on complete FULL sweeps). */ @ExtendWith(MockitoExtension.class) class PolicyRunnerTest { @Mock private PolicyEngine policyEngine; @Mock private InputSource folderSource; + @Mock private ProcessedLedger processedLedger; private final SourceStore sourceStore = new InProcessSourceStore(); private PolicyRunner runner; @@ -59,7 +65,8 @@ class PolicyRunnerTest { policyEngine, List.of(folderSource), sourceStore, - new InProcessSourceDocCounter()); + new InProcessSourceDocCounter(), + processedLedger); } @Test @@ -73,6 +80,9 @@ class PolicyRunnerTest { ArgumentCaptor inputs = ArgumentCaptor.forClass(PolicyInputs.class); verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any()); assertTrue(inputs.getValue().primary().isEmpty()); + // Ledger hygiene still runs: rows recorded for a generator policy's folder outputs + // are pruned by its own sweeps rather than accumulating until the policy is deleted. + verify(processedLedger).deleteUnseen(eq("p1"), anyLong()); } @Test @@ -80,7 +90,7 @@ class PolicyRunnerTest { InputSpec spec = InputSpec.folder("/in"); Policy policy = policy(List.of(spec)); when(folderSource.supports(spec)).thenReturn(true); - when(folderSource.resolve(spec)) + when(folderSource.resolve(eq(spec), any())) .thenReturn( List.of( ResolvedInput.of(PolicyInputs.of(List.of())), @@ -100,7 +110,7 @@ class PolicyRunnerTest { AtomicBoolean outcome = new AtomicBoolean(false); ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set); when(folderSource.supports(spec)).thenReturn(true); - when(folderSource.resolve(spec)).thenReturn(List.of(unit)); + when(folderSource.resolve(eq(spec), any())).thenReturn(List.of(unit)); CompletableFuture completion = new CompletableFuture<>(); when(policyEngine.runPolicy(any(), any(), any())) .thenReturn(new PolicyRunHandle("r", completion)); @@ -121,7 +131,7 @@ class PolicyRunnerTest { AtomicBoolean outcome = new AtomicBoolean(true); ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set); when(folderSource.supports(spec)).thenReturn(true); - when(folderSource.resolve(spec)).thenReturn(List.of(unit)); + when(folderSource.resolve(eq(spec), any())).thenReturn(List.of(unit)); CompletableFuture completion = new CompletableFuture<>(); when(policyEngine.runPolicy(any(), any(), any())) .thenReturn(new PolicyRunHandle("r", completion)); @@ -143,6 +153,98 @@ class PolicyRunnerTest { verifyNoInteractions(policyEngine); } + @Test + void aFullSweepStampsPresenceAndPrunesUnseenRows() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.listsExhaustively()).thenReturn(true); + when(folderSource.resolve(eq(spec), any())) + .thenAnswer( + invocation -> { + ResolveContext ctx = invocation.getArgument(1); + ctx.reportPresent(List.of("/in/a.pdf", "/in/b.pdf")); + return List.of(); + }); + + runner.run(policy); + + // Presence reporting also bulk-prefetches claim state: one lookup for the whole listing. + verify(processedLedger).statesFor(eq("p1"), eq(List.of("/in/a.pdf", "/in/b.pdf"))); + verify(processedLedger).markSeen("p1", Set.of("/in/a.pdf", "/in/b.pdf")); + verify(processedLedger).deleteUnseen(eq("p1"), anyLong()); + } + + @Test + void aLightSweepClaimsButSkipsLedgerHygiene() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(eq(spec), any())) + .thenReturn(List.of(ResolvedInput.of(PolicyInputs.of(List.of())))); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.run(policy, SweepKind.LIGHT); + + verify(policyEngine).runPolicy(eq(policy), any(), any()); + verify(processedLedger, never()).markSeen(any(), any()); + verify(processedLedger, never()).deleteUnseen(any(), anyLong()); + } + + @Test + void aSourceThatFailsToResolveVetoesCleanupButOthersStillRun() throws Exception { + InputSpec broken = InputSpec.folder("/broken"); + InputSpec healthy = InputSpec.folder("/healthy"); + Policy policy = policy(List.of(broken, healthy)); + when(folderSource.supports(any())).thenReturn(true); + when(folderSource.listsExhaustively()).thenReturn(true); + when(folderSource.resolve(eq(broken), any())).thenThrow(new IOException("mount gone")); + when(folderSource.resolve(eq(healthy), any())) + .thenReturn(List.of(ResolvedInput.of(PolicyInputs.of(List.of())))); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.run(policy); + + verify(policyEngine).runPolicy(eq(policy), any(), any()); // healthy source still ran + verify(processedLedger, never()).deleteUnseen(any(), anyLong()); // history preserved + } + + @Test + void aDisabledSourceVetoesCleanup() { + InputSpec spec = InputSpec.folder("/in"); + String pausedId = sourceStore.save(disabledSourceFrom(spec)).id(); + Policy policy = policyReferencing(List.of(pausedId)); + + runner.run(policy); + + verify(processedLedger, never()).deleteUnseen(any(), anyLong()); + } + + @Test + void aNonExhaustiveSourceVetoesCleanup() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.listsExhaustively()).thenReturn(false); + when(folderSource.resolve(eq(spec), any())).thenReturn(List.of()); + + runner.run(policy); + + verify(processedLedger, never()).deleteUnseen(any(), anyLong()); + } + + @Test + void aMissingSourceDoesNotVetoCleanup() { + // A deleted source's rows age out precisely because cleanup still runs. + Policy policy = policyReferencing(List.of("ghost-source-id")); + + runner.run(policy); + + verify(processedLedger).deleteUnseen(eq("p1"), anyLong()); + } + @Test void runWithSuppliedInputsBypassesSources() { Policy policy = policy(List.of(InputSpec.folder("/in"))); @@ -159,6 +261,10 @@ class PolicyRunnerTest { private Policy policy(List sources) { List sourceIds = sources.stream().map(spec -> sourceStore.save(sourceFrom(spec)).id()).toList(); + return policyReferencing(sourceIds); + } + + private static Policy policyReferencing(List sourceIds) { return new Policy( "p1", "p", @@ -173,4 +279,8 @@ class PolicyRunnerTest { private static Source sourceFrom(InputSpec spec) { return new Source(null, "src", spec.type(), spec.options(), true, "owner", null); } + + private static Source disabledSourceFrom(InputSpec spec) { + return new Source(null, "src", spec.type(), spec.options(), false, "owner", null); + } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java index 62bfa3aabb..64178ce9bb 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java @@ -1,17 +1,23 @@ package stirling.software.proprietary.policy.input; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -24,18 +30,26 @@ import org.springframework.core.env.StandardEnvironment; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.util.FileReadinessChecker; import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.source.InProcessSourceStore; -/** Tests for {@link FolderInputSource}: consume (claim + route) and snapshot (read-only) modes. */ +/** + * Tests for {@link FolderInputSource}: consume mode tracks files in place through the ledger, + * snapshot stays stateless, and discovery skips hidden entries and honours the recursive option. + */ @ExtendWith(MockitoExtension.class) class FolderInputSourceTest { + private static final String POLICY = "p1"; + @Mock private FileReadinessChecker readinessChecker; @TempDir Path tempDir; private FolderInputSource source; + private InProcessProcessedLedger ledger; + private RecordingContext ctx; @BeforeEach void setUp() { @@ -45,73 +59,279 @@ class FolderInputSourceTest { new FolderAccessGuard( properties, new StandardEnvironment(), new InProcessSourceStore()); source = new FolderInputSource(readinessChecker, guard); + ledger = new InProcessProcessedLedger(); + ctx = new RecordingContext(); // Lenient: the missing-dir / nonexistent-dir cases return before any readiness check. lenient().when(readinessChecker.isReady(any())).thenReturn(true); } @Test - void consumeClaimsFilesAndRoutesToDoneOnSuccess() throws IOException { + void consumeRemovesTheFileOnceProcessed() throws IOException { Path inputDir = Files.createDirectories(tempDir.resolve("in")); - Files.writeString(inputDir.resolve("doc.pdf"), "data"); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); - List work = source.resolve(InputSpec.folder(inputDir.toString())); + List work = source.resolve(InputSpec.folder(inputDir.toString()), ctx); assertEquals(1, work.size()); assertEquals(1, work.get(0).inputs().primary().size()); - // Claimed out of the input dir. - assertFalse(Files.exists(inputDir.resolve("doc.pdf"))); - assertTrue( - Files.exists( - inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf"))); + // In flight: still on disk, but a second sweep does not pick it up again. + assertTrue(Files.exists(file)); + assertTrue(Files.notExists(inputDir.resolve(".stirling"))); + assertTrue(source.resolve(InputSpec.folder(inputDir.toString()), ctx).isEmpty()); work.get(0).onComplete().accept(true); - assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("done").resolve("doc.pdf"))); - assertFalse( - Files.exists( - inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf"))); + assertTrue(Files.notExists(file)); + assertTrue(source.resolve(InputSpec.folder(inputDir.toString()), ctx).isEmpty()); } @Test - void consumeRoutesToErrorOnFailure() throws IOException { + void aFileReplacedMidRunSurvivesTheDeleteAndRunsAgain() throws IOException { Path inputDir = Files.createDirectories(tempDir.resolve("in")); - Files.writeString(inputDir.resolve("doc.pdf"), "data"); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); - List work = source.resolve(InputSpec.folder(inputDir.toString())); - work.get(0).onComplete().accept(false); + List work = source.resolve(InputSpec.folder(inputDir.toString()), ctx); + // The user saves a new version while the run is executing. + Files.writeString(file, "new data, different size"); + work.get(0).onComplete().accept(true); - assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("error").resolve("doc.pdf"))); + // The delete is version-guarded: the replacement is not the file that ran, so it stays + // and is claimed as fresh work instead of being marked processed. + assertTrue(Files.exists(file)); + assertEquals(1, source.resolve(InputSpec.folder(inputDir.toString()), ctx).size()); } @Test - void snapshotReadsWithoutClaiming() throws IOException { + void aSharedFileIsRemovedOnlyOnceEveryPolicyHasProcessedIt() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + InputSpec spec = InputSpec.folder(inputDir.toString()); + RecordingContext other = new RecordingContext("p2"); + + List mine = source.resolve(spec, ctx); + List theirs = source.resolve(spec, other); + assertEquals(1, mine.size()); + assertEquals(1, theirs.size()); + + mine.get(0).onComplete().accept(true); + // The other policy's claim is still in flight, so the first finisher must not delete. + assertTrue(Files.exists(file)); + + theirs.get(0).onComplete().accept(true); + assertTrue(Files.notExists(file)); + } + + @Test + void aSharedFileStaysParkedWhenAnyPolicyFailsOnIt() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + InputSpec spec = InputSpec.folder(inputDir.toString()); + RecordingContext other = new RecordingContext("p2"); + + List mine = source.resolve(spec, ctx); + List theirs = source.resolve(spec, other); + + theirs.get(0).onComplete().accept(false); + mine.get(0).onComplete().accept(true); + + // The failure parks the file for everyone (retried when it changes), regardless of + // which policy settled last. + assertTrue(Files.exists(file)); + assertTrue(source.resolve(spec, ctx).isEmpty()); + assertTrue(source.resolve(spec, other).isEmpty()); + } + + @Test + void aReDroppedFileIsProcessedAgain() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + + source.resolve(InputSpec.folder(inputDir.toString()), ctx).get(0).onComplete().accept(true); + Files.writeString(file, "data again"); + + assertEquals(1, source.resolve(InputSpec.folder(inputDir.toString()), ctx).size()); + } + + @Test + void aFailedFileStaysInPlaceAndIsNotRetriedUntilItChanges() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + + source.resolve(InputSpec.folder(inputDir.toString()), ctx) + .get(0) + .onComplete() + .accept(false); + + assertTrue(Files.exists(file)); + assertTrue(source.resolve(InputSpec.folder(inputDir.toString()), ctx).isEmpty()); + + Files.setLastModifiedTime(file, FileTime.from(Instant.now().plusSeconds(60))); + assertEquals(1, source.resolve(InputSpec.folder(inputDir.toString()), ctx).size()); + } + + @Test + void statModeRetriesAFailureOnATouchButHashModeDoesNot() throws IOException { + Path statDir = Files.createDirectories(tempDir.resolve("stat")); + Path hashDir = Files.createDirectories(tempDir.resolve("hash")); + Path statFile = statDir.resolve("doc.pdf"); + Path hashFile = hashDir.resolve("doc.pdf"); + Files.writeString(statFile, "data"); + Files.writeString(hashFile, "data"); + InputSpec statSpec = InputSpec.folder(statDir.toString()); + InputSpec hashSpec = + new InputSpec( + "folder", Map.of("directory", hashDir.toString(), "identity", "hash")); + + source.resolve(statSpec, ctx).get(0).onComplete().accept(false); + source.resolve(hashSpec, ctx).get(0).onComplete().accept(false); + + FileTime touched = FileTime.from(Instant.now().plusSeconds(60)); + Files.setLastModifiedTime(statFile, touched); + Files.setLastModifiedTime(hashFile, touched); + + // Same content, new mtime: stat mode calls that a new version and retries; hash mode + // verifies the content is unchanged and keeps the failure parked. + assertEquals(1, source.resolve(statSpec, ctx).size()); + assertTrue(source.resolve(hashSpec, ctx).isEmpty()); + } + + @Test + void hashModeRetriesAFailureOnARealContentChange() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Path file = inputDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + InputSpec spec = + new InputSpec( + "folder", Map.of("directory", inputDir.toString(), "identity", "hash")); + + source.resolve(spec, ctx).get(0).onComplete().accept(false); + Files.writeString(file, "data v2 - longer"); + + assertEquals(1, source.resolve(spec, ctx).size()); + } + + @Test + void snapshotReadsStatelesslyEverySweep() throws IOException { Path inputDir = Files.createDirectories(tempDir.resolve("in")); Files.writeString(inputDir.resolve("doc.pdf"), "data"); + InputSpec spec = + new InputSpec( + "folder", Map.of("directory", inputDir.toString(), "mode", "snapshot")); - List work = - source.resolve( - new InputSpec( - "folder", - Map.of("directory", inputDir.toString(), "mode", "snapshot"))); + List first = source.resolve(spec, ctx); + first.get(0).onComplete().accept(true); + List second = source.resolve(spec, ctx); + + assertEquals(1, first.size()); + assertEquals(1, second.size()); // no ledger involvement: every run sees the full set + assertTrue(ctx.present.isEmpty()); + assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); + } + + @Test + void hiddenFilesAndTheLegacyWorkDirAreIgnored() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + Files.writeString(inputDir.resolve(".hidden.pdf"), "secret"); + Path legacy = Files.createDirectories(inputDir.resolve(".stirling").resolve("done")); + Files.writeString(legacy.resolve("old.pdf"), "processed long ago"); + + List work = source.resolve(InputSpec.folder(inputDir.toString()), ctx); assertEquals(1, work.size()); - // Not moved, and completing the run is a no-op. - assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); - work.get(0).onComplete().accept(true); - assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); + assertEquals(1, ctx.present.size()); + assertTrue(ctx.present.get(0).endsWith("doc.pdf")); + } + + @Test + void recursiveDiscoversSubdirectoriesButNotHiddenOnes() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("top.pdf"), "a"); + Path sub = Files.createDirectories(inputDir.resolve("sub")); + Files.writeString(sub.resolve("nested.pdf"), "b"); + Path hiddenDir = Files.createDirectories(inputDir.resolve(".stirling")); + Files.writeString(hiddenDir.resolve("skipped.pdf"), "c"); + // Sink staging inside a watched subdirectory is pruned at any depth. + Path nestedStaging = Files.createDirectories(sub.resolve(".stirling").resolve("tmp")); + Files.writeString(nestedStaging.resolve("half-delivered"), "d"); + + InputSpec flat = InputSpec.folder(inputDir.toString()); + InputSpec recursive = + new InputSpec( + "folder", Map.of("directory", inputDir.toString(), "recursive", "true")); + + assertEquals(1, source.resolve(flat, ctx).size()); + assertEquals(1, source.resolve(recursive, ctx).size()); // top.pdf already claimed above + assertTrue(ctx.present.stream().anyMatch(identity -> identity.endsWith("nested.pdf"))); + assertTrue(ctx.present.stream().noneMatch(identity -> identity.endsWith("skipped.pdf"))); + assertTrue(ctx.present.stream().noneMatch(identity -> identity.endsWith("half-delivered"))); + } + + @Test + void unreadyFilesAreReportedPresentButNotClaimed() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Path file = inputDir.resolve("mid-write.pdf"); + Files.writeString(file, "partial"); + when(readinessChecker.isReady(file)).thenReturn(false); + + List work = source.resolve(InputSpec.folder(inputDir.toString()), ctx); + + assertTrue(work.isEmpty()); + // Reported present so a full sweep does not prune its row while it settles on disk. + assertEquals(1, ctx.present.size()); + assertTrue(ctx.present.get(0).endsWith("mid-write.pdf")); + } + + @Test + void nestedSourcesShareThePolicysLedgerAndDoNotDoubleClaim() throws IOException { + Path parent = Files.createDirectories(tempDir.resolve("in")); + Path child = Files.createDirectories(parent.resolve("sub")); + Files.writeString(child.resolve("doc.pdf"), "data"); + InputSpec parentRecursive = + new InputSpec( + "folder", Map.of("directory", parent.toString(), "recursive", "true")); + InputSpec childFlat = InputSpec.folder(child.toString()); + + // Same sweep, same policy context: whichever source resolves first wins the file. + assertEquals(1, source.resolve(parentRecursive, ctx).size()); + assertTrue(source.resolve(childFlat, ctx).isEmpty()); } @Test void missingDirectoryOptionFails() { assertThrows( IllegalArgumentException.class, - () -> source.resolve(new InputSpec("folder", Map.of()))); + () -> source.resolve(new InputSpec("folder", Map.of()), ctx)); } @Test - void nonexistentDirectoryYieldsNoWork() throws IOException { - List work = - source.resolve(InputSpec.folder(tempDir.resolve("nope").toString())); - assertTrue(work.isEmpty()); + void anUnknownIdentityModeIsRejected() { + assertThrows( + IllegalArgumentException.class, + () -> + source.validate( + new InputSpec( + "folder", + Map.of( + "directory", + tempDir.toString(), + "identity", + "guesswork")))); + } + + @Test + void nonexistentDirectoryFailsResolveSoTheSweepVetoesCleanup() { + // An unreachable directory (e.g. unmounted drive) must surface as a failed listing, not + // an empty one: the runner vetoes presence cleanup on failure, keeping the history that + // an empty listing would wipe. + assertThrows( + NoSuchFileException.class, + () -> source.resolve(InputSpec.folder(tempDir.resolve("nope").toString()), ctx)); } @Test @@ -126,7 +346,7 @@ class FolderInputSourceTest { Path outside = tempDir.resolveSibling("not-allowed"); assertThrows( IllegalArgumentException.class, - () -> source.resolve(InputSpec.folder(outside.toString()))); + () -> source.resolve(InputSpec.folder(outside.toString()), ctx)); assertThrows( IllegalArgumentException.class, () -> source.validate(InputSpec.folder(outside.toString()))); @@ -137,4 +357,40 @@ class FolderInputSourceTest { Path inputDir = tempDir.resolve("in"); assertEquals(List.of(inputDir), source.watchTargets(InputSpec.folder(inputDir.toString()))); } + + /** Policy-scoped context backed by the in-process ledger, recording presence reports. */ + private class RecordingContext implements ResolveContext { + + private final String policyId; + private final List present = new ArrayList<>(); + + private RecordingContext() { + this(POLICY); + } + + private RecordingContext(String policyId) { + this.policyId = policyId; + } + + @Override + public boolean claim(String identity, String gate, Supplier contentHash) { + return ledger.claim(policyId, identity, gate, contentHash); + } + + @Override + public void settle( + String identity, String finalGate, String finalContentHash, boolean success) { + ledger.settle(policyId, identity, finalGate, finalContentHash, success); + } + + @Override + public boolean allSettledDone(String identity) { + return ledger.allSettledDone(identity); + } + + @Override + public void reportPresent(Collection identities) { + present.addAll(identities); + } + } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/FolderIdentitiesTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/FolderIdentitiesTest.java new file mode 100644 index 0000000000..9f56411d9b --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/FolderIdentitiesTest.java @@ -0,0 +1,84 @@ +package stirling.software.proprietary.policy.ledger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link FolderIdentities}: identity derivation must agree for the same file, even + * through a symlinked alias of the directory. + */ +class FolderIdentitiesTest { + + @TempDir Path tempDir; + + @Test + void identityAgreesAcrossASymlinkedAliasOfTheDirectory() throws IOException { + Path real = Files.createDirectories(tempDir.resolve("real")); + Path alias = Files.createSymbolicLink(tempDir.resolve("alias"), real); + Files.writeString(real.resolve("doc.pdf"), "data"); + + String viaReal = + FolderIdentities.identity( + FolderIdentities.canonicalDir(real), real, real.resolve("doc.pdf")); + String viaAlias = + FolderIdentities.identity( + FolderIdentities.canonicalDir(alias), alias, alias.resolve("doc.pdf")); + + assertEquals(viaReal, viaAlias); + } + + @Test + void identityOfANestedFileKeepsItsRelativePath() throws IOException { + Path dir = Files.createDirectories(tempDir.resolve("in")); + Path nested = Files.createDirectories(dir.resolve("sub")).resolve("doc.pdf"); + Files.writeString(nested, "data"); + + String identity = + FolderIdentities.identity(FolderIdentities.canonicalDir(dir), dir, nested); + + assertTrue(identity.endsWith("sub" + java.io.File.separator + "doc.pdf")); + } + + @Test + void theGateTracksSizeAndMtime() throws IOException { + Path file = tempDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + String before = FolderIdentities.statGate(file); + + Files.setLastModifiedTime(file, FileTime.from(Instant.now().plusSeconds(60))); + + assertNotEquals(before, FolderIdentities.statGate(file)); + } + + @Test + void theContentHashIgnoresMtimeButTracksContent() throws IOException { + Path file = tempDir.resolve("doc.pdf"); + Files.writeString(file, "data"); + String before = FolderIdentities.contentHash(file); + + Files.setLastModifiedTime(file, FileTime.from(Instant.now().plusSeconds(60))); + assertEquals(before, FolderIdentities.contentHash(file)); + + Files.writeString(file, "different"); + assertNotEquals(before, FolderIdentities.contentHash(file)); + } + + @Test + void identityHashIsAStableFixedWidthKey() { + String hash = IdentityHasher.identityHash("/in/doc.pdf"); + + assertEquals(64, hash.length()); + assertEquals(hash, IdentityHasher.identityHash("/in/doc.pdf")); + assertNotEquals(hash, IdentityHasher.identityHash("/in/other.pdf")); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/InProcessProcessedLedgerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/InProcessProcessedLedgerTest.java new file mode 100644 index 0000000000..08ac10061d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/InProcessProcessedLedgerTest.java @@ -0,0 +1,12 @@ +package stirling.software.proprietary.policy.ledger; + +import java.util.function.Supplier; + +/** {@link InProcessProcessedLedger} against the shared {@link ProcessedLedger} contract. */ +class InProcessProcessedLedgerTest extends ProcessedLedgerContractTest { + + @Override + ProcessedLedger newLedger(Supplier nowMillis) { + return new InProcessProcessedLedger(nowMillis); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/JpaProcessedLedgerDbTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/JpaProcessedLedgerDbTest.java new file mode 100644 index 0000000000..8c1bd85c94 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/JpaProcessedLedgerDbTest.java @@ -0,0 +1,36 @@ +package stirling.software.proprietary.policy.ledger; + +import java.util.function.Supplier; + +import org.junit.jupiter.api.AfterEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigurationPackage; +import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest; + +/** + * {@link JpaProcessedLedger} against the shared contract on a real (H2) database. The inherited + * tests run outside {@code @DataJpaTest}'s per-test transaction (the transaction attribute resolves + * against the declaring class, the plain contract base), so every ledger call commits in its own + * transaction as at runtime; state is wiped explicitly instead of relying on rollback. + */ +@DataJpaTest +class JpaProcessedLedgerDbTest extends ProcessedLedgerContractTest { + + @Autowired private ProcessedFileRepository repository; + + @AfterEach + void wipeLedger() { + // deleteAll() skips entities whose isNew() is hardcoded true, so use the bulk form. + repository.deleteAllInBatch(); + } + + @Override + ProcessedLedger newLedger(Supplier nowMillis) { + return new JpaProcessedLedger(repository, nowMillis); + } + + @SpringBootConfiguration + @AutoConfigurationPackage + static class TestApp {} +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/ProcessedLedgerContractTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/ProcessedLedgerContractTest.java new file mode 100644 index 0000000000..800e8a6bde --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/ledger/ProcessedLedgerContractTest.java @@ -0,0 +1,360 @@ +package stirling.software.proprietary.policy.ledger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** The {@link ProcessedLedger} contract, run against every implementation so they cannot drift. */ +abstract class ProcessedLedgerContractTest { + + static final String POLICY = "p1"; + static final String OTHER_POLICY = "p2"; + static final String FILE = "/in/doc.pdf"; + static final String GATE = "100:1111"; + static final String NEW_GATE = "100:2222"; + static final String HASH = "hash-aaa"; + static final String NEW_HASH = "hash-bbb"; + + final AtomicLong clock = new AtomicLong(1_000_000L); + + ProcessedLedger ledger; + + abstract ProcessedLedger newLedger(Supplier nowMillis); + + @BeforeEach + void createLedger() { + ledger = newLedger(clock::get); + } + + @Test + void aFileIsClaimedOnceAndSkippedWhileInFlight() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, null)); // new version waits for settle + } + + @Test + void aSettledFileIsSkippedAtTheSameGate() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.settle(POLICY, FILE, GATE, null, true); + + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + } + + @Test + void aMovedGateIsReclaimedInGateOnlyMode() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.settle(POLICY, FILE, GATE, null, true); + + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); + } + + @Test + void settlingAtTheOutputsVersionStopsAnInPlaceOverwriteLooping() { + ledger.claim(POLICY, FILE, GATE, null); + // The run overwrote the input; settle re-reads and lands on the produced version. + ledger.settle(POLICY, FILE, NEW_GATE, null, true); + + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, null)); // own output: skip + assertTrue(ledger.claim(POLICY, FILE, "100:3333", null)); // later user edit: reprocess + } + + @Test + void aFailedFileIsNotRetriedUntilItChanges() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.settle(POLICY, FILE, GATE, null, false); + + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); + } + + @Test + void aTouchedButUnchangedFileRefreshesTheGateInsteadOfReprocessing() { + ledger.claim(POLICY, FILE, GATE, hash(HASH)); + ledger.settle(POLICY, FILE, GATE, HASH, true); + + // Same content under a new gate (touch / identical re-copy): verified, not reprocessed. + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH))); + + // The gate was refreshed, so the next sweep takes the cheap path: no content read at all. + CountingSupplier counting = new CountingSupplier(HASH); + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, counting)); + assertEquals(0, counting.invocations.get()); + } + + @Test + void theVerificationTierIsNotConsultedWhileTheGateMatches() { + ledger.claim(POLICY, FILE, GATE, hash(HASH)); + ledger.settle(POLICY, FILE, GATE, HASH, true); + + CountingSupplier counting = new CountingSupplier(HASH); + assertFalse(ledger.claim(POLICY, FILE, GATE, counting)); + assertEquals(0, counting.invocations.get()); + } + + @Test + void aRealContentChangeUnderANewGateIsReprocessed() { + ledger.claim(POLICY, FILE, GATE, hash(HASH)); + ledger.settle(POLICY, FILE, GATE, HASH, true); + + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, hash(NEW_HASH))); + } + + @Test + void aGateOnlySettledRowCannotBeContentVerifiedSoItReprocesses() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.settle(POLICY, FILE, GATE, null, true); + + // The row stored no hash, so "same content" is unprovable: reprocess on gate change. + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH))); + } + + @Test + void aFailedFileStaysParkedThroughATouch() { + ledger.claim(POLICY, FILE, GATE, hash(HASH)); + ledger.settle(POLICY, FILE, GATE, HASH, false); + + // A touch must not resurrect an ERROR row; only a real content change does. + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH))); + assertTrue(ledger.claim(POLICY, FILE, "100:3333", hash(NEW_HASH))); + } + + @Test + void interruptedRunsAreRetriedABoundedNumberOfTimes() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // attempt 1 dies with the JVM + ledger.recoverInterrupted(); + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // attempt 2 + ledger.recoverInterrupted(); + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // attempt 3, the last + ledger.recoverInterrupted(); + + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); // parked: no crash-loop + } + + @Test + void aNewGateResetsTheInterruptRetryBudgetInGateOnlyMode() { + for (int attempt = 0; attempt < ProcessedLedger.MAX_ATTEMPTS; attempt++) { + ledger.claim(POLICY, FILE, GATE, null); + ledger.recoverInterrupted(); + } + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); + ledger.recoverInterrupted(); + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); // fresh budget at the new version + } + + @Test + void aTouchDoesNotResetTheInterruptRetryBudgetWhenContentIsVerified() { + assertTrue(ledger.claim(POLICY, FILE, GATE, hash(HASH))); // attempt 1 + ledger.recoverInterrupted(); + // Same content, moved gate: still the interrupted work, still bounded. + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH))); // attempt 2 + ledger.recoverInterrupted(); + assertTrue(ledger.claim(POLICY, FILE, "100:3333", hash(HASH))); // attempt 3 + ledger.recoverInterrupted(); + + assertFalse(ledger.claim(POLICY, FILE, "100:4444", hash(HASH))); // parked + assertTrue(ledger.claim(POLICY, FILE, "100:5555", hash(NEW_HASH))); // real change: fresh + } + + @Test + void recoveryOnlyTouchesInFlightRows() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.settle(POLICY, FILE, GATE, null, true); + ledger.recoverInterrupted(); + + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); // still DONE, not retried + } + + @Test + void anOutputIsSkippedByItsProducerButSeenByOtherPolicies() { + ledger.recordOutput(POLICY, FILE, GATE, HASH); + + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); // producer skips its own output + assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null)); // chaining still works + } + + @Test + void anOutputIsSkippedByAHashVerifyingProducerEvenIfTheGateMoved() { + ledger.recordOutput(POLICY, FILE, GATE, HASH); + + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH))); + } + + @Test + void policiesTrackTheSameFileIndependently() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null)); + } + + @Test + void statesForSnapshotsOnlyExistingRows() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + + Map states = ledger.statesFor(POLICY, List.of(FILE, "/in/other.pdf")); + + assertEquals(1, states.size()); + assertEquals(ProcessedFileStatus.PROCESSING, states.get(FILE).status()); + assertEquals(GATE, states.get(FILE).gate()); + } + + @Test + void aStaleAbsentSnapshotLosesTheClaimRaceInsteadOfDoubleClaiming() { + ClaimState absent = ledger.statesFor(POLICY, List.of(FILE)).get(FILE); // no row yet + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // another sweep wins meanwhile + + assertFalse(ledger.claim(POLICY, FILE, GATE, null, absent)); + } + + @Test + void aStaleSettledSnapshotCannotReclaimAnInFlightRow() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + ledger.settle(POLICY, FILE, GATE, null, true); + ClaimState settled = ledger.statesFor(POLICY, List.of(FILE)).get(FILE); + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); // a fresh sweep reclaims first + + assertFalse(ledger.claim(POLICY, FILE, "100:3333", null, settled)); + } + + @Test + void aForgottenOutputIsClaimableAtAnyVersion() { + ledger.recordOutput(POLICY, FILE, GATE, HASH); + ledger.forgetOutput(POLICY, FILE, GATE); + + // Even a byte-identical file at that identity is fresh work: the record is gone. + assertTrue(ledger.claim(POLICY, FILE, GATE, hash(HASH))); + } + + @Test + void forgetOutputLeavesARowReclaimedInTheMeantime() { + ledger.recordOutput(POLICY, FILE, GATE, HASH); + assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); // a real claim took the row over + + ledger.forgetOutput(POLICY, FILE, GATE); + + assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, null)); // still in flight, not deleted + } + + @Test + void deletionConsensusNeedsEveryClaimantSettledDone() { + assertTrue(ledger.allSettledDone(FILE)); // vacuous: no rows yet + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null)); + ledger.settle(POLICY, FILE, GATE, null, true); + assertFalse(ledger.allSettledDone(FILE)); // the other claim is still in flight + ledger.settle(OTHER_POLICY, FILE, GATE, null, true); + assertTrue(ledger.allSettledDone(FILE)); + } + + @Test + void aFailedClaimVetoesDeletionConsensus() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null)); + ledger.settle(POLICY, FILE, GATE, null, true); + ledger.settle(OTHER_POLICY, FILE, GATE, null, false); + assertFalse(ledger.allSettledDone(FILE)); + } + + @Test + void anInterruptedClaimVetoesDeletionConsensus() { + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + ledger.recoverInterrupted(); + assertFalse(ledger.allSettledDone(FILE)); + } + + @Test + void settleRecreatesARowRemovedMidRun() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.clearPolicy(POLICY); // e.g. a clear-history while the run is in flight + ledger.settle(POLICY, FILE, GATE, null, true); + + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + } + + @Test + void presenceCleanupRemovesOnlyUnseenSettledRows() { + String inFlight = "/in/in-flight.pdf"; + String stillPresent = "/in/still-present.pdf"; + String deleted = "/in/deleted.pdf"; + ledger.claim(POLICY, inFlight, GATE, null); + ledger.recordOutput(POLICY, stillPresent, GATE, HASH); + ledger.recordOutput(POLICY, deleted, GATE, HASH); + + clock.addAndGet(10_000); + long sweepStart = clock.get(); + ledger.markSeen(POLICY, List.of(inFlight, stillPresent)); // deleted.pdf is gone from disk + assertEquals(1, ledger.deleteUnseen(POLICY, sweepStart)); + + assertFalse(ledger.claim(POLICY, inFlight, GATE, null)); // in-flight row survived + assertFalse(ledger.claim(POLICY, stillPresent, GATE, null)); // stamped row survived + assertTrue(ledger.claim(POLICY, deleted, GATE, null)); // forgotten: a re-drop reprocesses + } + + @Test + void presenceCleanupNeverRemovesInFlightRowsEvenUnstamped() { + ledger.claim(POLICY, FILE, GATE, null); + clock.addAndGet(10_000); + + assertEquals(0, ledger.deleteUnseen(POLICY, clock.get())); + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + } + + @Test + void rowsWrittenDuringTheSweepSurviveItsCleanup() { + long sweepStart = clock.get(); + // Recorded after the sweep's cutoff: unseen by it, but newer, so it must survive. + clock.addAndGet(5); + ledger.recordOutput(POLICY, FILE, GATE, HASH); + + assertEquals(0, ledger.deleteUnseen(POLICY, sweepStart)); + assertFalse(ledger.claim(POLICY, FILE, GATE, null)); + } + + @Test + void markSeenOnUnknownIdentitiesIsANoOp() { + ledger.markSeen(POLICY, List.of("/never/claimed.pdf")); + assertEquals(0, ledger.deleteUnseen(POLICY, clock.get())); + } + + @Test + void clearPolicyForgetsOnlyThatPolicy() { + ledger.claim(POLICY, FILE, GATE, null); + ledger.settle(POLICY, FILE, GATE, null, true); + ledger.claim(OTHER_POLICY, FILE, GATE, null); + ledger.settle(OTHER_POLICY, FILE, GATE, null, true); + + ledger.clearPolicy(POLICY); + + assertTrue(ledger.claim(POLICY, FILE, GATE, null)); + assertFalse(ledger.claim(OTHER_POLICY, FILE, GATE, null)); + } + + static Supplier hash(String value) { + return () -> value; + } + + static final class CountingSupplier implements Supplier { + final AtomicInteger invocations = new AtomicInteger(); + private final String value; + + CountingSupplier(String value) { + this.value = value; + } + + @Override + public String get() { + invocations.incrementAndGet(); + return value; + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java index f1b8b599ac..f9e1c92403 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java @@ -10,6 +10,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,24 +22,35 @@ import org.springframework.core.io.Resource; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.job.ResultFile; import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.ledger.FolderIdentities; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; import stirling.software.proprietary.policy.model.OutputSpec; import stirling.software.proprietary.policy.source.InProcessSourceStore; -/** Tests for {@link FolderOutputSink}: outputs are written to the configured directory on disk. */ +/** + * Tests for {@link FolderOutputSink}: outputs are staged hidden, recorded in the ledger, then + * atomically renamed into the configured directory. + */ class FolderOutputSinkTest { + private static final OutputDelivery AD_HOC = new OutputDelivery("run-1", null); + private static final OutputDelivery POLICY_RUN = new OutputDelivery("run-1", "p1"); + @TempDir Path tempDir; private FolderOutputSink sink; + private InProcessProcessedLedger ledger; @BeforeEach void setUp() { ApplicationProperties properties = new ApplicationProperties(); properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString())); + ledger = new InProcessProcessedLedger(); sink = new FolderOutputSink( new FolderAccessGuard( - properties, new StandardEnvironment(), new InProcessSourceStore())); + properties, new StandardEnvironment(), new InProcessSourceStore()), + ledger); } @Test @@ -46,13 +58,80 @@ class FolderOutputSinkTest { Path out = tempDir.resolve("out"); List outputs = List.of(named("a.pdf", "aaa"), named("b.pdf", "bb")); - List results = - sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + List results = sink.deliver(AD_HOC, outputs, OutputSpec.folder(out.toString())); assertEquals(2, results.size()); assertTrue(Files.exists(out.resolve("a.pdf"))); assertEquals("aaa", Files.readString(out.resolve("a.pdf"))); assertEquals("bb", Files.readString(out.resolve("b.pdf"))); + // Nothing left behind in the staging dir. + try (Stream staged = Files.list(out.resolve(".stirling").resolve("tmp"))) { + assertEquals(0, staged.count()); + } + } + + @Test + void recordsThePolicysOutputsSoOnlyOtherPoliciesReprocessThem() throws IOException { + Path out = tempDir.resolve("out"); + + sink.deliver(POLICY_RUN, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString())); + + Path delivered = FolderIdentities.canonicalDir(out).resolve("a.pdf"); + String gate = FolderIdentities.statGate(delivered); + assertFalse(ledger.claim("p1", delivered.toString(), gate, null)); // producer skips it + assertTrue(ledger.claim("p2", delivered.toString(), gate, null)); // chaining still works + } + + @Test + void aHashVerifyingProducerSkipsItsOwnOutputEvenIfTheGateMoved() throws IOException { + Path out = tempDir.resolve("out"); + + sink.deliver(POLICY_RUN, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString())); + + // A hash-verifying reader matches on content even when the stat moved. + Path delivered = FolderIdentities.canonicalDir(out).resolve("a.pdf"); + assertFalse( + ledger.claim( + "p1", + delivered.toString(), + "999:12345", + () -> { + try { + return FolderIdentities.contentHash(delivered); + } catch (IOException e) { + throw new java.io.UncheckedIOException(e); + } + })); + } + + @Test + void recordsAnOutputBeforeItBecomesVisible() throws IOException { + Path out = tempDir.resolve("out"); + VisibilityAssertingLedger orderedLedger = new VisibilityAssertingLedger(); + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString())); + FolderOutputSink orderedSink = + new FolderOutputSink( + new FolderAccessGuard( + properties, new StandardEnvironment(), new InProcessSourceStore()), + orderedLedger); + + orderedSink.deliver( + POLICY_RUN, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString())); + + assertTrue(orderedLedger.recorded); + assertTrue(Files.exists(out.resolve("a.pdf"))); + } + + @Test + void adHocDeliveriesRecordNothing() throws IOException { + Path out = tempDir.resolve("out"); + + sink.deliver(AD_HOC, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString())); + + Path delivered = FolderIdentities.canonicalDir(out).resolve("a.pdf"); + // No row was recorded, so any policy (including a hypothetical producer) may claim it. + assertTrue(ledger.claim("p1", delivered.toString(), "any-gate", null)); } @Test @@ -60,7 +139,7 @@ class FolderOutputSinkTest { Path out = tempDir.resolve("out"); List outputs = List.of(named("a.pdf", "first"), named("a.pdf", "second")); - sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + sink.deliver(AD_HOC, outputs, OutputSpec.folder(out.toString())); assertTrue(Files.exists(out.resolve("a.pdf"))); assertTrue(Files.exists(out.resolve("a (1).pdf"))); @@ -72,7 +151,7 @@ class FolderOutputSinkTest { assertThrows(IllegalArgumentException.class, () -> sink.validate(noDir)); assertThrows( IllegalArgumentException.class, - () -> sink.deliver("run-1", List.of(named("a.pdf", "x")), noDir)); + () -> sink.deliver(AD_HOC, List.of(named("a.pdf", "x")), noDir)); } @Test @@ -81,7 +160,7 @@ class FolderOutputSinkTest { assertThrows(IllegalArgumentException.class, () -> sink.validate(outside)); assertThrows( IllegalArgumentException.class, - () -> sink.deliver("run-1", List.of(named("a.pdf", "x")), outside)); + () -> sink.deliver(AD_HOC, List.of(named("a.pdf", "x")), outside)); } @Test @@ -90,7 +169,7 @@ class FolderOutputSinkTest { List outputs = List.of(named("../escape.pdf", "x"), named("nested/deep.pdf", "y")); - sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + sink.deliver(AD_HOC, outputs, OutputSpec.folder(out.toString())); // Each name is reduced to its bare form inside the target dir; nothing escapes. assertTrue(Files.exists(out.resolve("escape.pdf"))); @@ -106,4 +185,20 @@ class FolderOutputSinkTest { } }; } + + /** Fails the delivery if an output is visible at its final path before being recorded. */ + private static class VisibilityAssertingLedger extends InProcessProcessedLedger { + + private boolean recorded; + + @Override + public synchronized void recordOutput( + String policyId, String identity, String gate, String contentHash) { + assertFalse( + Files.exists(Path.of(identity)), + "output must be recorded before it is visible at its final path"); + recorded = true; + super.recordOutput(policyId, identity, gate, contentHash); + } + } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java index 58c5123327..2c3adf691e 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java @@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.trigger; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -26,6 +27,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.engine.SweepKind; import stirling.software.proprietary.policy.input.InputSource; import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.OutputSpec; @@ -100,8 +102,8 @@ class FolderWatchTriggerTest { trigger.runForChangedDirs(Set.of(normalized("/in/a"))); - verify(policyRunner).run(a); - verify(policyRunner, never()).run(b); + verify(policyRunner).run(a, SweepKind.LIGHT); + verify(policyRunner, never()).run(eq(b), any()); } @Test @@ -112,8 +114,8 @@ class FolderWatchTriggerTest { trigger.runForChangedDirs(Set.of(normalized("/in/a"))); - verify(policyRunner).run(good); - verify(policyRunner, never()).run(bad); + verify(policyRunner).run(good, SweepKind.LIGHT); + verify(policyRunner, never()).run(eq(bad), any()); } @Test diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/InitialSecuritySetupTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/InitialSecuritySetupTest.java index adf9486094..3e4c75f645 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/InitialSecuritySetupTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/InitialSecuritySetupTest.java @@ -26,6 +26,7 @@ import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.security.service.DatabaseServiceInterface; import stirling.software.proprietary.security.service.SaveUserRequest; +import stirling.software.proprietary.security.service.TeamMembershipService; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; import stirling.software.proprietary.service.UserLicenseSettingsService; @@ -38,6 +39,7 @@ class InitialSecuritySetupTest { @Mock private DatabaseServiceInterface databaseService; @Mock private UserLicenseSettingsService licenseSettingsService; @Mock private Environment environment; + @Mock private TeamMembershipService teamMembershipService; private ApplicationProperties applicationProperties; private InitialSecuritySetup initialSecuritySetup; @@ -63,7 +65,8 @@ class InitialSecuritySetupTest { applicationProperties, databaseService, licenseSettingsService, - environment); + environment, + teamMembershipService); } @Test diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerLoginTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerLoginTest.java index 6a27b4d578..6586fc11b6 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerLoginTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerLoginTest.java @@ -29,6 +29,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.enumeration.Role; import stirling.software.proprietary.access.service.ResourceAccessService; +import stirling.software.proprietary.access.service.TeamLeadLookup; import stirling.software.proprietary.security.model.AuthenticationType; import stirling.software.proprietary.security.model.Authority; import stirling.software.proprietary.security.model.User; @@ -60,6 +61,7 @@ class AuthControllerLoginTest { @Mock private TotpService totpService; @Mock private RefreshRateLimitService refreshRateLimitService; @Mock private ResourceAccessService resourceAccessService; + @Mock private TeamLeadLookup teamLeadLookup; @BeforeEach void setUp() { @@ -83,7 +85,8 @@ class AuthControllerLoginTest { securityProperties, applicationProperties, new stirling.software.proprietary.service.AiUserDataService(null), - resourceAccessService); + resourceAccessService, + teamLeadLookup); mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerMoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerMoreTest.java index 611929b5b9..aaf931f597 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerMoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AuthControllerMoreTest.java @@ -29,6 +29,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.enumeration.Role; import stirling.software.proprietary.access.service.ResourceAccessService; +import stirling.software.proprietary.access.service.TeamLeadLookup; import stirling.software.proprietary.security.model.AuthenticationType; import stirling.software.proprietary.security.model.Authority; import stirling.software.proprietary.security.model.User; @@ -61,6 +62,7 @@ class AuthControllerMoreTest { @Mock private TotpService totpService; @Mock private RefreshRateLimitService refreshRateLimitService; @Mock private ResourceAccessService resourceAccessService; + @Mock private TeamLeadLookup teamLeadLookup; @BeforeEach void setUp() { @@ -84,7 +86,8 @@ class AuthControllerMoreTest { securityProperties, applicationProperties, new stirling.software.proprietary.service.AiUserDataService(null), - resourceAccessService); + resourceAccessService, + teamLeadLookup); mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerMoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerMoreTest.java index 75c23b4630..ac2219efa3 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerMoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerMoreTest.java @@ -30,6 +30,7 @@ import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.service.EmailService; import stirling.software.proprietary.security.service.LoginAttemptService; +import stirling.software.proprietary.security.service.TeamMembershipService; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; import stirling.software.proprietary.security.session.SessionPersistentRegistry; @@ -46,6 +47,7 @@ class UserControllerMoreTest { @Mock private EmailService emailService; @Mock private UserLicenseSettingsService licenseSettingsService; @Mock private LoginAttemptService loginAttemptService; + @Mock private TeamMembershipService teamMembershipService; private ApplicationProperties applicationProperties; private MockMvc mockMvc; @@ -64,7 +66,8 @@ class UserControllerMoreTest { userRepository, Optional.of(emailService), licenseSettingsService, - loginAttemptService); + loginAttemptService, + teamMembershipService); mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerTest.java index 7bcc0da9f4..b3a1e8de93 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerTest.java @@ -33,6 +33,7 @@ import stirling.software.proprietary.security.model.api.user.UsernameAndPass; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.service.EmailService; import stirling.software.proprietary.security.service.LoginAttemptService; +import stirling.software.proprietary.security.service.TeamMembershipService; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; import stirling.software.proprietary.security.session.SessionPersistentRegistry; @@ -53,6 +54,7 @@ class UserControllerTest { @Mock private EmailService emailService; @Mock private UserLicenseSettingsService licenseSettingsService; @Mock private LoginAttemptService loginAttemptService; + @Mock private TeamMembershipService teamMembershipService; private ApplicationProperties applicationProperties; private MockMvc mockMvc; @@ -72,7 +74,8 @@ class UserControllerTest { userRepository, Optional.of(emailService), licenseSettingsService, - loginAttemptService); + loginAttemptService, + teamMembershipService); mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/TeamMembershipServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/TeamMembershipServiceTest.java new file mode 100644 index 0000000000..68bd2d1727 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/TeamMembershipServiceTest.java @@ -0,0 +1,159 @@ +package stirling.software.proprietary.security.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; + +@ExtendWith(MockitoExtension.class) +class TeamMembershipServiceTest { + + @Mock private TeamMembershipRepository membershipRepository; + @Mock private org.springframework.core.env.Environment environment; + + @InjectMocks private TeamMembershipService service; + + @org.junit.jupiter.api.BeforeEach + void notSaas() { + org.mockito.Mockito.lenient() + .when(environment.getActiveProfiles()) + .thenReturn(new String[] {}); + } + + @Test + void syncCreatesMemberRowForUsersTeam() { + User user = userInTeam(5, 7); + when(membershipRepository.findByUserId(5L)).thenReturn(List.of()); + + service.syncMembership(user); + + ArgumentCaptor captor = ArgumentCaptor.forClass(TeamMembership.class); + verify(membershipRepository).save(captor.capture()); + assertThat(captor.getValue().getTeam().getId()).isEqualTo(7L); + assertThat(captor.getValue().getRole()).isEqualTo(TeamRole.MEMBER); + assertThat(captor.getValue().getInvitedAt()).isNotNull(); + } + + @Test + void syncMovesRowsWhenUserChangedTeam() { + User user = userInTeam(5, 8); + TeamMembership oldRow = row(7, user, TeamRole.LEADER); + when(membershipRepository.findByUserId(5L)).thenReturn(List.of(oldRow)); + + service.syncMembership(user); + + verify(membershipRepository).delete(oldRow); + ArgumentCaptor captor = ArgumentCaptor.forClass(TeamMembership.class); + verify(membershipRepository).save(captor.capture()); + assertThat(captor.getValue().getTeam().getId()).isEqualTo(8L); + assertThat(captor.getValue().getRole()).isEqualTo(TeamRole.MEMBER); + } + + @Test + void syncPreservesLeaderRoleOnSameTeam() { + User user = userInTeam(5, 7); + TeamMembership existing = row(7, user, TeamRole.LEADER); + when(membershipRepository.findByUserId(5L)).thenReturn(List.of(existing)); + + service.syncMembership(user); + + verify(membershipRepository, never()).delete(any()); + verify(membershipRepository, never()).save(any()); + } + + @Test + void syncRemovesAllRowsWhenUserHasNoTeam() { + User user = new User(); + user.setId(5L); + TeamMembership stale = row(7, user, TeamRole.MEMBER); + when(membershipRepository.findByUserId(5L)).thenReturn(List.of(stale)); + + service.syncMembership(user); + + verify(membershipRepository).delete(stale); + verify(membershipRepository, never()).save(any()); + } + + @Test + void setOwnerPromotesExistingRow() { + User user = userInTeam(5, 7); + TeamMembership existing = row(7, user, TeamRole.MEMBER); + when(membershipRepository.findByTeamIdAndUserId(7L, 5L)).thenReturn(Optional.of(existing)); + + service.setOwner(user.getTeam(), user); + + assertThat(existing.getRole()).isEqualTo(TeamRole.LEADER); + verify(membershipRepository).save(existing); + } + + @Test + void setOwnerCreatesLeaderRowWhenMissing() { + User user = userInTeam(5, 7); + when(membershipRepository.findByTeamIdAndUserId(7L, 5L)).thenReturn(Optional.empty()); + + service.setOwner(user.getTeam(), user); + + ArgumentCaptor captor = ArgumentCaptor.forClass(TeamMembership.class); + verify(membershipRepository).save(captor.capture()); + assertThat(captor.getValue().getRole()).isEqualTo(TeamRole.LEADER); + } + + @Test + void removeOwnerDemotesToMemberAndKeepsRow() { + User user = userInTeam(5, 7); + TeamMembership existing = row(7, user, TeamRole.LEADER); + when(membershipRepository.findByTeamIdAndUserId(7L, 5L)).thenReturn(Optional.of(existing)); + + service.removeOwner(user.getTeam(), user); + + assertThat(existing.getRole()).isEqualTo(TeamRole.MEMBER); + verify(membershipRepository).save(existing); + verify(membershipRepository, never()).delete(any()); + } + + @Test + void deleteAllForUserDropsRowsAndInvitationRefs() { + User user = userInTeam(5, 7); + + service.deleteAllForUser(user); + + verify(membershipRepository).deleteByUserId(5L); + verify(membershipRepository).clearInvitedBy(user); + } + + private User userInTeam(long userId, long teamId) { + User u = new User(); + u.setId(userId); + Team t = new Team(); + t.setId(teamId); + u.setTeam(t); + return u; + } + + private TeamMembership row(long teamId, User user, TeamRole role) { + Team t = new Team(); + t.setId(teamId); + TeamMembership m = new TeamMembership(); + m.setTeam(t); + m.setUser(user); + m.setRole(role); + return m; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceMoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceMoreTest.java index cfb0f2e885..bc516172a6 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceMoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceMoreTest.java @@ -71,6 +71,12 @@ class UserServiceMoreTest { @Mock private FileShareRepository fileShareRepository; @Mock private FileShareAccessRepository fileShareAccessRepository; + @Mock + private stirling.software.proprietary.integration.repository.IntegrationConfigRepository + integrationConfigRepository; + + @Mock private TeamMembershipService teamMembershipService; + @InjectMocks private UserService userService; @AfterEach diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceTest.java index 713fd4b9c3..95ec72d962 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceTest.java @@ -24,6 +24,7 @@ import org.springframework.security.crypto.password.PasswordEncoder; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.enumeration.Role; import stirling.software.common.model.exception.UnsupportedProviderException; +import stirling.software.proprietary.access.repository.ResourceGrantRepository; import stirling.software.proprietary.model.Team; import stirling.software.proprietary.security.database.repository.AuthorityRepository; import stirling.software.proprietary.security.database.repository.PersistentLoginRepository; @@ -63,6 +64,13 @@ class UserServiceTest { @Mock private StorageCleanupEntryRepository storageCleanupEntryRepository; @Mock private FileShareRepository fileShareRepository; @Mock private FileShareAccessRepository fileShareAccessRepository; + @Mock private ResourceGrantRepository resourceGrantRepository; + + @Mock + private stirling.software.proprietary.integration.repository.IntegrationConfigRepository + integrationConfigRepository; + + @Mock private TeamMembershipService teamMembershipService; @Spy @InjectMocks private UserService userService; diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index 7dc174c041..15c07bab71 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -2,6 +2,7 @@ package stirling.software.proprietary.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -230,6 +231,57 @@ class AiWorkflowServiceTest { // 1:1 mapping preserves each input's filename. assertEquals("a.pdf", result.getResultFiles().get(0).getFileName()); assertEquals("b.pdf", result.getResultFiles().get(1).getFileName()); + // Each output points back at the input it came from so the client versions it in place. + assertEquals(0, result.getResultFiles().get(0).getSourceIndex()); + assertEquals(1, result.getResultFiles().get(1).getSourceIndex()); + } + + @Test + void mergeOutputHasNoSourceIndex() throws IOException { + MockMultipartFile a = pdf("a.pdf", "a-bytes"); + MockMultipartFile b = pdf("b.pdf", "b-bytes"); + stubOrchestrator( + """ + {"outcome":"tool_call","tool":"%s","parameters":{},"rationale":"Merging"} + """ + .formatted(MERGE_ENDPOINT)); + when(toolMetadataService.isMultiInput(MERGE_ENDPOINT)).thenReturn(true); + when(toolMetadataService.shouldUnpackZipResponse(MERGE_ENDPOINT)).thenReturn(false); + stubEndpoint(MERGE_ENDPOINT, pdfResource("merged-bytes", "merged.pdf")); + stubFileStorage(); + + AiWorkflowResponse result = + service.orchestrate(requestFor(new MockMultipartFile[] {a, b}, "merge these")); + + // A merge draws on several inputs, so there is no single source to version in place. + assertNull(result.getResultFiles().get(0).getSourceIndex()); + } + + @Test + void splitOutputsHaveNoSourceIndex() throws IOException { + MockMultipartFile input = pdf("doc.pdf", "original"); + stubOrchestrator( + """ + {"outcome":"tool_call","tool":"%s","parameters":{},"rationale":"Splitting"} + """ + .formatted(SPLIT_ENDPOINT)); + when(toolMetadataService.isMultiInput(SPLIT_ENDPOINT)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(SPLIT_ENDPOINT)).thenReturn(true); + stubEndpoint( + SPLIT_ENDPOINT, + zipResource( + "doc.zip", + List.of( + new ZipEntryBytes("page-1.pdf", "page-one"), + new ZipEntryBytes("page-2.pdf", "page-two")))); + stubFileStorage(); + + AiWorkflowResponse result = service.orchestrate(requestFor(input, "split")); + + // One input fanned out to many outputs, so none is a clean 1:1 version β€” the client adds + // them as fresh files and leaves the original in place. + assertNull(result.getResultFiles().get(0).getSourceIndex()); + assertNull(result.getResultFiles().get(1).getSourceIndex()); } @Test diff --git a/app/saas/src/main/java/stirling/software/saas/accountlink/AccountLinkController.java b/app/saas/src/main/java/stirling/software/saas/accountlink/AccountLinkController.java index 91974a6d04..490b3d5db1 100644 --- a/app/saas/src/main/java/stirling/software/saas/accountlink/AccountLinkController.java +++ b/app/saas/src/main/java/stirling/software/saas/accountlink/AccountLinkController.java @@ -20,10 +20,10 @@ import io.swagger.v3.oas.annotations.Hidden; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; -import stirling.software.saas.repository.TeamMembershipRepository; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/main/java/stirling/software/saas/controller/SaasTeamController.java b/app/saas/src/main/java/stirling/software/saas/controller/SaasTeamController.java index 6253d64651..9346e18b4b 100644 --- a/app/saas/src/main/java/stirling/software/saas/controller/SaasTeamController.java +++ b/app/saas/src/main/java/stirling/software/saas/controller/SaasTeamController.java @@ -20,15 +20,15 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.annotations.api.TeamApi; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; import stirling.software.saas.model.TeamInvitation; -import stirling.software.saas.model.TeamMembership; import stirling.software.saas.repository.TeamInvitationRepository; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.security.TeamSecurityExpressions; import stirling.software.saas.service.SaasTeamExtensionService; import stirling.software.saas.service.SaasTeamService; diff --git a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygInvoicesController.java b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygInvoicesController.java index 6b8bb2c883..80c20902ce 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygInvoicesController.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygInvoicesController.java @@ -20,13 +20,13 @@ import io.swagger.v3.oas.annotations.Hidden; import lombok.extern.slf4j.Slf4j; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.payg.policy.PaygTeamExtensions; import stirling.software.saas.payg.repository.PaygTeamExtensionsRepository; import stirling.software.saas.payg.stripe.StripeInvoiceDao; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygPaymentMethodController.java b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygPaymentMethodController.java index 724d6715d3..89ab18230f 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygPaymentMethodController.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygPaymentMethodController.java @@ -18,13 +18,13 @@ import io.swagger.v3.oas.annotations.Hidden; import lombok.extern.slf4j.Slf4j; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.payg.policy.PaygTeamExtensions; import stirling.software.saas.payg.repository.PaygTeamExtensionsRepository; import stirling.software.saas.payg.stripe.StripePaymentMethodDao; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java index b79d57117e..90616bea55 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/api/PaygWalletController.java @@ -32,9 +32,10 @@ import jakarta.validation.constraints.Min; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.payg.api.WalletSnapshotResponse.ActivityRow; import stirling.software.saas.payg.api.WalletSnapshotResponse.CategoryBreakdown; import stirling.software.saas.payg.api.WalletSnapshotResponse.MemberRow; @@ -51,7 +52,6 @@ import stirling.software.saas.payg.repository.WalletLedgerRepository; import stirling.software.saas.payg.repository.WalletPolicyRepository; import stirling.software.saas.payg.wallet.WalletLedgerEntry; import stirling.software.saas.payg.wallet.WalletPolicy; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/main/java/stirling/software/saas/procurement/api/ProcurementController.java b/app/saas/src/main/java/stirling/software/saas/procurement/api/ProcurementController.java index f92267da33..57f7e6a3a3 100644 --- a/app/saas/src/main/java/stirling/software/saas/procurement/api/ProcurementController.java +++ b/app/saas/src/main/java/stirling/software/saas/procurement/api/ProcurementController.java @@ -25,16 +25,16 @@ import io.swagger.v3.oas.annotations.Hidden; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.enumeration.TeamRole; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.procurement.config.ProcurementConfigurationProperties; import stirling.software.saas.procurement.model.ProcurementDeal; import stirling.software.saas.procurement.model.ProcurementQuote; import stirling.software.saas.procurement.pricing.QuoteConfig; import stirling.software.saas.procurement.pricing.QuoteLineItem; import stirling.software.saas.procurement.service.ProcurementService; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/main/java/stirling/software/saas/procurement/service/ProcurementService.java b/app/saas/src/main/java/stirling/software/saas/procurement/service/ProcurementService.java index 57f530cc9a..225e0327e8 100644 --- a/app/saas/src/main/java/stirling/software/saas/procurement/service/ProcurementService.java +++ b/app/saas/src/main/java/stirling/software/saas/procurement/service/ProcurementService.java @@ -17,7 +17,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.enumeration.TeamRole; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.procurement.config.ProcurementConfigurationProperties; import stirling.software.saas.procurement.license.EnterpriseLicenseService; import stirling.software.saas.procurement.license.LicenseEntitlements; @@ -28,7 +29,6 @@ import stirling.software.saas.procurement.pricing.QuoteBreakdown; import stirling.software.saas.procurement.pricing.QuoteConfig; import stirling.software.saas.procurement.repository.ProcurementDealRepository; import stirling.software.saas.procurement.repository.ProcurementQuoteRepository; -import stirling.software.saas.repository.TeamMembershipRepository; /** * Orchestrates a linked team's procurement journey: start a (mock-licensed) trial, build a diff --git a/app/saas/src/main/java/stirling/software/saas/security/SaasPortalAuditScopeResolver.java b/app/saas/src/main/java/stirling/software/saas/security/SaasPortalAuditScopeResolver.java index 00c7c42757..0d0df35688 100644 --- a/app/saas/src/main/java/stirling/software/saas/security/SaasPortalAuditScopeResolver.java +++ b/app/saas/src/main/java/stirling/software/saas/security/SaasPortalAuditScopeResolver.java @@ -11,7 +11,7 @@ import lombok.RequiredArgsConstructor; import stirling.software.proprietary.audit.PortalAuditScope; import stirling.software.proprietary.audit.PortalAuditScopeResolver; -import stirling.software.saas.repository.TeamMembershipRepository; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; /** SaaS audit visibility: admins see the server; team LEADERs see their team (by member email). */ @Component diff --git a/app/saas/src/main/java/stirling/software/saas/security/SaasPrincipalResolver.java b/app/saas/src/main/java/stirling/software/saas/security/SaasPrincipalResolver.java new file mode 100644 index 0000000000..d064c7b59d --- /dev/null +++ b/app/saas/src/main/java/stirling/software/saas/security/SaasPrincipalResolver.java @@ -0,0 +1,30 @@ +package stirling.software.saas.security; + +import java.util.HashSet; +import java.util.Set; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +import stirling.software.proprietary.access.model.PrincipalRef; +import stirling.software.proprietary.access.service.PrincipalResolver; +import stirling.software.proprietary.security.model.User; + +/** USER/TEAM only: SaaS has no org concept, so an ORG grant must never match across tenants. */ +@Component +@Profile("saas") +public class SaasPrincipalResolver implements PrincipalResolver { + + @Override + public Set principalsOf(User user) { + if (user == null) { + return Set.of(); + } + Set principals = new HashSet<>(); + principals.add(PrincipalRef.user(user.getId())); + if (user.getTeam() != null) { + principals.add(PrincipalRef.team(user.getTeam().getId())); + } + return principals; + } +} diff --git a/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java b/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java index 3dfdf541cc..5263705986 100644 --- a/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java +++ b/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java @@ -12,8 +12,8 @@ import lombok.RequiredArgsConstructor; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.service.UserService; -import stirling.software.saas.repository.TeamMembershipRepository; /** * Security expressions for team-based authorization in saas mode. Wired into @@ -65,6 +65,12 @@ public class TeamSecurityExpressions { return currentUser.getTeam().getId(); } + /** The current authenticated user's id, or {@code null} if unauthenticated. */ + public Long currentUserId() { + User currentUser = getCurrentUser(); + return currentUser == null ? null : currentUser.getId(); + } + /** Whether the current authenticated user is any kind of member of the given team. */ public boolean isTeamMember(Long teamId) { User currentUser = getCurrentUser(); diff --git a/app/saas/src/main/java/stirling/software/saas/service/SaasTeamService.java b/app/saas/src/main/java/stirling/software/saas/service/SaasTeamService.java index 6d3e42e806..94996ca01c 100644 --- a/app/saas/src/main/java/stirling/software/saas/service/SaasTeamService.java +++ b/app/saas/src/main/java/stirling/software/saas/service/SaasTeamService.java @@ -16,17 +16,17 @@ import stirling.software.common.model.enumeration.InvitationStatus; import stirling.software.common.model.enumeration.Role; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.saas.accountlink.LinkedInstanceRepository; import stirling.software.saas.billing.repository.BillingSubscriptionRepository; import stirling.software.saas.config.SupabaseConfigurationProperties; import stirling.software.saas.model.TeamInvitation; -import stirling.software.saas.model.TeamMembership; import stirling.software.saas.repository.SaasTeamExtensionsRepository; import stirling.software.saas.repository.TeamInvitationRepository; -import stirling.software.saas.repository.TeamMembershipRepository; /** SaaS-only team management: invitations, personal teams, seat caps, paid-subscription gating. */ @Service @@ -48,6 +48,17 @@ public class SaasTeamService { private final SaasTeamExtensionsRepository saasTeamExtensionsRepository; private final LinkedInstanceRepository linkedInstanceRepository; private final stirling.software.proprietary.security.service.UserService userService; + private final stirling.software.proprietary.access.repository.ResourceGrantRepository + resourceGrantRepository; + private final stirling.software.proprietary.integration.repository.IntegrationConfigRepository + integrationConfigRepository; + + // Team-owned integration configs + team grants FK the teams row; purge before deleting a team. + private void purgeTeamOwnedResources(Long teamId) { + integrationConfigRepository.deleteByOwnerTeam_Id(teamId); + resourceGrantRepository.deleteByPrincipalTypeAndPrincipalId( + stirling.software.proprietary.access.model.PrincipalType.TEAM, teamId); + } public static final String DEFAULT_TEAM_NAME = "Default"; public static final String INTERNAL_TEAM_NAME = "Internal"; @@ -347,6 +358,7 @@ public class SaasTeamService { "Deleting empty personal team {} after user {} joined another team", teamToDelete.getId(), acceptingUser.getUsername()); + purgeTeamOwnedResources(teamToDelete.getId()); teamRepository.delete(teamToDelete); } @@ -437,6 +449,7 @@ public class SaasTeamService { if (!saasTeamExtensionService.isPersonal(team) && membershipRepository.countByTeamId(teamId) == 0) { log.info("Deleting empty non-personal team {} after last member removed", teamId); + purgeTeamOwnedResources(team.getId()); teamRepository.delete(team); } @@ -551,6 +564,7 @@ public class SaasTeamService { if (!saasTeamExtensionService.isPersonal(team) && membershipRepository.countByTeamId(teamId) == 0) { log.info("Deleting empty non-personal team {} after last member left", teamId); + purgeTeamOwnedResources(team.getId()); teamRepository.delete(team); } diff --git a/app/saas/src/main/resources/db/migration/saas/V30__classification_labels.sql b/app/saas/src/main/resources/db/migration/saas/V30__classification_labels.sql new file mode 100644 index 0000000000..0719d73fb1 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V30__classification_labels.sql @@ -0,0 +1,13 @@ +-- Classification labels (gated by policies.enabled): the flat multi-label vocabulary the document +-- classifier runs against. One admin-editable row per team. The whole label set lives as JSON in +-- labels_json (authoritative on read). team_id is a natural key and a plain value (not a foreign +-- key) to stay decoupled from the security entities, so classification can be enabled or disabled +-- without touching them; the sentinel 0 holds the unteamed (login-disabled) team set. Hibernate +-- ddl-auto would also create this, but this keeps the schema explicit for Flyway-managed deploys. + +CREATE TABLE IF NOT EXISTS classification_labels ( + team_id BIGINT PRIMARY KEY, + labels_json TEXT, + updated_at TIMESTAMP, + updated_by VARCHAR(255) +); diff --git a/app/saas/src/main/resources/db/migration/saas/V31__policy_sort_order.sql b/app/saas/src/main/resources/db/migration/saas/V31__policy_sort_order.sql new file mode 100644 index 0000000000..2b2bdbd25b --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V31__policy_sort_order.sql @@ -0,0 +1,5 @@ +-- Team-wide policy run order (see PolicyEntity.sortOrder). The order policies run in is stored +-- server-side and shared by the whole team (not per-user in the browser), and is admin-editable. +-- Nullable so existing rows keep working; reads treat a null as 0 (coalesce), and the store +-- appends a new policy at max(order)+1 so setting one up adds it to the end of the queue. +ALTER TABLE policies ADD COLUMN sort_order INTEGER; diff --git a/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql b/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql new file mode 100644 index 0000000000..4be3fa07d8 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql @@ -0,0 +1,36 @@ +-- Per-policy processed-file ledger: +-- +-- policy_processed_files one row per (policy, file identity) recording the version a policy +-- last settled that file at, so folder sources track files in place +-- instead of moving them into a work directory. signature is a cheap +-- version gate (folder: size:mtime); content_hash an optional strong +-- token consulted only when the gate moves. Rows are claimed into +-- PROCESSING, settled to DONE/ERROR, flipped to INTERRUPTED at boot if +-- a run died with the JVM, and pruned once the file is gone from all of +-- the policy's sources, so the table stays near the set of files +-- currently present. +-- +-- Gated by policies.enabled like the rest of the subsystem; Hibernate ddl-auto would also create +-- this, but the migration keeps the schema explicit for the Flyway-managed deployments. + +CREATE TABLE IF NOT EXISTS policy_processed_files ( + policy_id VARCHAR(255) NOT NULL, + identity_hash VARCHAR(64) NOT NULL, + identity VARCHAR(4096), + signature VARCHAR(255) NOT NULL, + content_hash VARCHAR(64), + status VARCHAR(16) NOT NULL, + attempts SMALLINT NOT NULL DEFAULT 1, + last_seen BIGINT NOT NULL DEFAULT 0, + updated_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (policy_id, identity_hash) +); + +CREATE INDEX IF NOT EXISTS idx_processed_files_policy_seen + ON policy_processed_files (policy_id, last_seen); + +-- The cross-policy deletion-consensus check (existsByIdentityHashAndStatusNot) filters identity_hash +-- alone, so it cannot use the (policy_id, identity_hash) primary key; it runs once per successfully +-- consumed file, so index it to avoid a full scan on the hot path. +CREATE INDEX IF NOT EXISTS idx_processed_files_identity + ON policy_processed_files (identity_hash); diff --git a/app/saas/src/test/java/stirling/software/saas/accountlink/AccountLinkControllerTest.java b/app/saas/src/test/java/stirling/software/saas/accountlink/AccountLinkControllerTest.java index 0de790cac0..e12888edb3 100644 --- a/app/saas/src/test/java/stirling/software/saas/accountlink/AccountLinkControllerTest.java +++ b/app/saas/src/test/java/stirling/software/saas/accountlink/AccountLinkControllerTest.java @@ -19,12 +19,12 @@ import org.springframework.security.core.authority.SimpleGrantedAuthority; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.accountlink.AccountLinkController.RegisterRequest; import stirling.software.saas.accountlink.AccountLinkController.RegisterResponse; -import stirling.software.saas.model.TeamMembership; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/test/java/stirling/software/saas/controller/SaasTeamControllerTest.java b/app/saas/src/test/java/stirling/software/saas/controller/SaasTeamControllerTest.java index ea8721a4c3..4dd8a9670d 100644 --- a/app/saas/src/test/java/stirling/software/saas/controller/SaasTeamControllerTest.java +++ b/app/saas/src/test/java/stirling/software/saas/controller/SaasTeamControllerTest.java @@ -36,8 +36,10 @@ import stirling.software.common.model.enumeration.InvitationStatus; import stirling.software.common.model.enumeration.Role; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.proprietary.security.service.TeamService; import stirling.software.proprietary.security.service.UserService; @@ -45,9 +47,7 @@ import stirling.software.saas.controller.SaasTeamController.InviteUserRequest; import stirling.software.saas.controller.SaasTeamController.RenameTeamRequest; import stirling.software.saas.controller.SaasTeamController.UpdateSeatsRequest; import stirling.software.saas.model.TeamInvitation; -import stirling.software.saas.model.TeamMembership; import stirling.software.saas.repository.TeamInvitationRepository; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.security.TeamSecurityExpressions; import stirling.software.saas.service.SaasTeamExtensionService; import stirling.software.saas.service.SaasTeamService; diff --git a/app/saas/src/test/java/stirling/software/saas/payg/api/PaygInvoicesControllerTest.java b/app/saas/src/test/java/stirling/software/saas/payg/api/PaygInvoicesControllerTest.java index 9266b02336..dc29ceb22d 100644 --- a/app/saas/src/test/java/stirling/software/saas/payg/api/PaygInvoicesControllerTest.java +++ b/app/saas/src/test/java/stirling/software/saas/payg/api/PaygInvoicesControllerTest.java @@ -23,14 +23,14 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.authority.SimpleGrantedAuthority; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.payg.api.PaygInvoicesController.InvoiceResponse; import stirling.software.saas.payg.policy.PaygTeamExtensions; import stirling.software.saas.payg.repository.PaygTeamExtensionsRepository; import stirling.software.saas.payg.stripe.StripeInvoiceDao; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.util.AuthenticationUtils; /** diff --git a/app/saas/src/test/java/stirling/software/saas/payg/api/PaygPaymentMethodControllerTest.java b/app/saas/src/test/java/stirling/software/saas/payg/api/PaygPaymentMethodControllerTest.java index c4c0780b86..b7b7561a88 100644 --- a/app/saas/src/test/java/stirling/software/saas/payg/api/PaygPaymentMethodControllerTest.java +++ b/app/saas/src/test/java/stirling/software/saas/payg/api/PaygPaymentMethodControllerTest.java @@ -24,15 +24,15 @@ import org.springframework.security.oauth2.jwt.Jwt; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.payg.api.PaygPaymentMethodController.PaymentMethodResponse; import stirling.software.saas.payg.policy.PaygTeamExtensions; import stirling.software.saas.payg.repository.PaygTeamExtensionsRepository; import stirling.software.saas.payg.stripe.StripePaymentMethodDao; import stirling.software.saas.payg.stripe.StripePaymentMethodDao.CardSummary; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.security.EnhancedJwtAuthenticationToken; /** diff --git a/app/saas/src/test/java/stirling/software/saas/payg/api/PaygWalletControllerTest.java b/app/saas/src/test/java/stirling/software/saas/payg/api/PaygWalletControllerTest.java index 6cc97d916b..0e02c0ce50 100644 --- a/app/saas/src/test/java/stirling/software/saas/payg/api/PaygWalletControllerTest.java +++ b/app/saas/src/test/java/stirling/software/saas/payg/api/PaygWalletControllerTest.java @@ -31,9 +31,10 @@ import org.springframework.security.oauth2.jwt.Jwt; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.User; -import stirling.software.saas.model.TeamMembership; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.saas.payg.api.PaygWalletController.UpdateCapRequest; import stirling.software.saas.payg.api.WalletSnapshotResponse.MemberRow; import stirling.software.saas.payg.billing.TeamBillingContext; @@ -50,7 +51,6 @@ import stirling.software.saas.payg.repository.PaygTeamExtensionsRepository; import stirling.software.saas.payg.repository.WalletLedgerRepository; import stirling.software.saas.payg.repository.WalletPolicyRepository; import stirling.software.saas.payg.wallet.WalletPolicy; -import stirling.software.saas.repository.TeamMembershipRepository; import stirling.software.saas.security.EnhancedJwtAuthenticationToken; /** diff --git a/app/saas/src/test/java/stirling/software/saas/security/SaasPrincipalResolverTest.java b/app/saas/src/test/java/stirling/software/saas/security/SaasPrincipalResolverTest.java new file mode 100644 index 0000000000..20acc83e5b --- /dev/null +++ b/app/saas/src/test/java/stirling/software/saas/security/SaasPrincipalResolverTest.java @@ -0,0 +1,39 @@ +package stirling.software.saas.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.access.model.PrincipalRef; +import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.security.model.User; + +class SaasPrincipalResolverTest { + + private final SaasPrincipalResolver resolver = new SaasPrincipalResolver(); + + @Test + void projectsUserAndTeamOnlyNeverOrg() { + User u = new User(); + u.setId(5L); + Team t = new Team(); + t.setId(7L); + u.setTeam(t); + + assertThat(resolver.principalsOf(u)) + .containsExactlyInAnyOrder(PrincipalRef.user(5L), PrincipalRef.team(7L)); + } + + @Test + void userWithoutTeamProjectsUserOnly() { + User u = new User(); + u.setId(5L); + + assertThat(resolver.principalsOf(u)).containsExactly(PrincipalRef.user(5L)); + } + + @Test + void nullUserProjectsNothing() { + assertThat(resolver.principalsOf(null)).isEmpty(); + } +} diff --git a/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsMoreTest.java b/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsMoreTest.java index 580623c878..364db2855d 100644 --- a/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsMoreTest.java +++ b/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsMoreTest.java @@ -25,10 +25,10 @@ import org.springframework.security.oauth2.jwt.Jwt; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.service.UserService; -import stirling.software.saas.model.TeamMembership; -import stirling.software.saas.repository.TeamMembershipRepository; /** * Additional branch coverage for {@link TeamSecurityExpressions}: the JWT resolution path, the diff --git a/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsTest.java b/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsTest.java index 23226e4467..c698daccca 100644 --- a/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsTest.java +++ b/app/saas/src/test/java/stirling/software/saas/security/TeamSecurityExpressionsTest.java @@ -20,10 +20,10 @@ import org.springframework.security.core.context.SecurityContextHolder; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.service.UserService; -import stirling.software.saas.model.TeamMembership; -import stirling.software.saas.repository.TeamMembershipRepository; /** * {@link TeamSecurityExpressions#isCurrentUserTeamLeader()} β€” used to gate policy editing on SaaS. diff --git a/app/saas/src/test/java/stirling/software/saas/service/SaasTeamServiceTest.java b/app/saas/src/test/java/stirling/software/saas/service/SaasTeamServiceTest.java index ac2a4cecba..5f1083cb3f 100644 --- a/app/saas/src/test/java/stirling/software/saas/service/SaasTeamServiceTest.java +++ b/app/saas/src/test/java/stirling/software/saas/service/SaasTeamServiceTest.java @@ -30,18 +30,18 @@ import stirling.software.common.model.enumeration.InvitationStatus; import stirling.software.common.model.enumeration.Role; import stirling.software.common.model.enumeration.TeamRole; import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamMembership; import stirling.software.proprietary.security.database.repository.UserRepository; import stirling.software.proprietary.security.model.Authority; import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.repository.TeamMembershipRepository; import stirling.software.proprietary.security.repository.TeamRepository; import stirling.software.saas.accountlink.LinkedInstanceRepository; import stirling.software.saas.billing.repository.BillingSubscriptionRepository; import stirling.software.saas.config.SupabaseConfigurationProperties; import stirling.software.saas.model.TeamInvitation; -import stirling.software.saas.model.TeamMembership; import stirling.software.saas.repository.SaasTeamExtensionsRepository; import stirling.software.saas.repository.TeamInvitationRepository; -import stirling.software.saas.repository.TeamMembershipRepository; /** * Unit tests for {@link SaasTeamService}. @@ -68,6 +68,14 @@ class SaasTeamServiceTest { @Mock private LinkedInstanceRepository linkedInstanceRepository; @Mock private stirling.software.proprietary.security.service.UserService userService; + @Mock + private stirling.software.proprietary.access.repository.ResourceGrantRepository + resourceGrantRepository; + + @Mock + private stirling.software.proprietary.integration.repository.IntegrationConfigRepository + integrationConfigRepository; + @InjectMocks private SaasTeamService service; private static final UUID SUPABASE_ID = UUID.fromString("11111111-2222-3333-4444-555555555555"); diff --git a/engine/src/stirling/agents/__init__.py b/engine/src/stirling/agents/__init__.py index c22bd6c977..7bc2cb63a2 100644 --- a/engine/src/stirling/agents/__init__.py +++ b/engine/src/stirling/agents/__init__.py @@ -1,5 +1,6 @@ """Agent modules for Stirling AI reasoning flows.""" +from .document_classifier import DocumentClassifierAgent from .execution import ExecutionPlanningAgent from .orchestrator import OrchestratorAgent from .pdf_create import PdfCreateAgent @@ -9,6 +10,7 @@ from .pdf_review import PdfReviewAgent from .user_spec import UserSpecAgent __all__ = [ + "DocumentClassifierAgent", "ExecutionPlanningAgent", "OrchestratorAgent", "PdfCreateAgent", diff --git a/engine/src/stirling/agents/document_classifier.py b/engine/src/stirling/agents/document_classifier.py new file mode 100644 index 0000000000..bb4ab5ef00 --- /dev/null +++ b/engine/src/stirling/agents/document_classifier.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import logging + +from pydantic import Field +from pydantic_ai import Agent +from pydantic_ai.output import NativeOutput + +from stirling.contracts import ( + ClassifyDocumentRequest, + ClassifyDocumentResponse, + DocumentClassificationResponse, + LabelOption, + PageText, +) +from stirling.models import ApiModel +from stirling.services import AppRuntime + +logger = logging.getLogger(__name__) + +# A document carries at most this many labels; anything past the cap is dropped +# in the model's order. +MAX_ASSIGNED_LABELS = 5 +# Pages read from each end of the document. A document's type is evident from +# its opening (and closing) pages, so a fixed window keeps cost and latency flat +# regardless of length. Promote to AppSettings if it ever needs tuning. +WINDOW_PAGES = 2 + + +_SYSTEM_PROMPT = ( + "You identify what a document is by assigning labels, choosing only from a " + "fixed list of allowed labels you are given.\n" + "\n" + "Rules:\n" + f"- Pick up to {MAX_ASSIGNED_LABELS} labels that describe this document's type.\n" + "- Only use labels from the allowed list, spelled exactly as listed.\n" + "- Return an empty list if none fit.\n" + "- Judge from the document's content and structure, not from keywords alone. " + "The document may be in any language.\n" + "- You are shown only the first and last pages; that is enough to identify the type." +) + + +class _ClassifierOutput(ApiModel): + """Raw model answer, before it is validated against the allowed vocabulary.""" + + labels: list[str] = Field( + default_factory=list, + description="Labels from the allowed list that describe this document's type.", + ) + + +def render_labels(labels: list[LabelOption]) -> str: + """Render the allowed vocabulary for the prompt β€” the model reasons over the + human names (it never sees ids).""" + names = [label.name for label in labels] + return f"Allowed labels: {', '.join(names) or '(none)'}" + + +def select_window(pages: list[PageText], window: int = WINDOW_PAGES) -> list[PageText]: + """Return the first and last ``window`` pages, never overlapping. + + Documents short enough that the two ends would meet are returned whole. The + caller usually sends just the window already; this is a defensive trim in + case it sends more. + """ + if window <= 0 or len(pages) <= window * 2: + return list(pages) + return [*pages[:window], *pages[-window:]] + + +def format_window(pages: list[PageText]) -> str: + if not pages: + return "(no extractable text)" + return "\n\n".join(f"[Page {page.page_number}]\n{page.text}" for page in pages) + + +def validate_labels(output: _ClassifierOutput, allowed: list[LabelOption]) -> DocumentClassificationResponse: + """Coerce a raw model answer (names) onto the allowed vocabulary, returning + label **ids**. + + The model answers with names; they are matched case-insensitively to the + allowed vocabulary and returned as that label's id. Anything off-list is + dropped, duplicates collapse to the first occurrence, and the result is + capped at ``MAX_ASSIGNED_LABELS`` in the model's order. The model identifies; + these rules decide what is allowed to stand. + """ + id_by_lower_name = {label.name.lower(): label.id for label in allowed} + + kept: list[str] = [] + for name in output.labels: + label_id = id_by_lower_name.get(name.strip().lower()) + if label_id is not None and label_id not in kept: + kept.append(label_id) + if len(kept) == MAX_ASSIGNED_LABELS: + break + + return DocumentClassificationResponse(labels=kept) + + +class DocumentClassifierAgent: + """Assigns labels to a document from an allowed vocabulary. + + Reads the bounded page window supplied on the request (first/last + ``WINDOW_PAGES``) and runs a single fast-model pass, then validates the + answer against the vocabulary so nothing off-list survives. + """ + + def __init__(self, runtime: AppRuntime) -> None: + self.runtime = runtime + self._agent = Agent( + model=runtime.fast_model, + output_type=NativeOutput(_ClassifierOutput), + system_prompt=_SYSTEM_PROMPT, + model_settings=runtime.fast_model_settings, + ) + + async def classify(self, request: ClassifyDocumentRequest) -> ClassifyDocumentResponse: + # The caller (the backend) always supplies the allowed vocabulary β€” the + # team's stored labels β€” so the engine holds no vocabulary of its own. + allowed = request.labels + window = select_window(request.pages) + prompt = self._build_prompt(request.file_name, allowed, window) + logger.debug("[classify] prompt:\n%s", prompt) + result = await self._agent.run(prompt) + return validate_labels(result.output, allowed) + + @staticmethod + def _build_prompt(file_name: str, allowed: list[LabelOption], window: list[PageText]) -> str: + return ( + f"{render_labels(allowed)}\n\n" + f"Document file name: {file_name}\n" + f"Document content (first and last pages):\n{format_window(window)}" + ) diff --git a/engine/src/stirling/api/app.py b/engine/src/stirling/api/app.py index 7e6272c28b..34b0da36fd 100644 --- a/engine/src/stirling/api/app.py +++ b/engine/src/stirling/api/app.py @@ -10,6 +10,7 @@ from pydantic_ai import Agent from pydantic_ai.models.instrumented import InstrumentationSettings from stirling.agents import ( + DocumentClassifierAgent, ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, @@ -24,6 +25,7 @@ from stirling.api.middleware import UserIdMiddleware from stirling.api.routes import ( agent_capabilities_router, agent_draft_router, + document_classifier_router, document_router, execution_router, ledger_router, @@ -95,6 +97,7 @@ async def lifespan(fast_api: FastAPI): fast_api.state.execution_planning_agent = ExecutionPlanningAgent(runtime) fast_api.state.math_auditor_agent = MathAuditorAgent(runtime) fast_api.state.pdf_comment_agent = PdfCommentAgent(runtime) + fast_api.state.document_classifier_agent = DocumentClassifierAgent(runtime) tracer_provider = setup_posthog_tracking(settings) if tracer_provider: Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider)) @@ -131,6 +134,7 @@ app.include_router(document_router, dependencies=_user_gate) app.include_router(ledger_router, dependencies=_user_gate) app.include_router(pdf_comments_router, dependencies=_user_gate) app.include_router(agent_capabilities_router, dependencies=_user_gate) +app.include_router(document_classifier_router, dependencies=_user_gate) @app.get("/health", response_model=HealthResponse) diff --git a/engine/src/stirling/api/dependencies.py b/engine/src/stirling/api/dependencies.py index 780e97c9d7..ed9aa3f041 100644 --- a/engine/src/stirling/api/dependencies.py +++ b/engine/src/stirling/api/dependencies.py @@ -5,6 +5,7 @@ from typing import Annotated from fastapi import Depends, HTTPException, Request, status from stirling.agents import ( + DocumentClassifierAgent, ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, @@ -55,6 +56,10 @@ def get_pdf_comment_agent(request: Request) -> PdfCommentAgent: return request.app.state.pdf_comment_agent +def get_document_classifier_agent(request: Request) -> DocumentClassifierAgent: + return request.app.state.document_classifier_agent + + def require_user_id() -> UserId: """FastAPI dependency for routes that touch per-user storage. diff --git a/engine/src/stirling/api/routes/__init__.py b/engine/src/stirling/api/routes/__init__.py index 1c1e4e1d23..f40e9918f8 100644 --- a/engine/src/stirling/api/routes/__init__.py +++ b/engine/src/stirling/api/routes/__init__.py @@ -1,5 +1,6 @@ from .agent_capabilities import router as agent_capabilities_router from .agent_drafts import router as agent_draft_router +from .document_classifier import router as document_classifier_router from .documents import router as document_router from .execution import router as execution_router from .ledger import router as ledger_router @@ -11,6 +12,7 @@ from .pdf_questions import router as pdf_question_router __all__ = [ "agent_capabilities_router", "agent_draft_router", + "document_classifier_router", "document_router", "execution_router", "ledger_router", diff --git a/engine/src/stirling/api/routes/document_classifier.py b/engine/src/stirling/api/routes/document_classifier.py new file mode 100644 index 0000000000..649c0f4e9e --- /dev/null +++ b/engine/src/stirling/api/routes/document_classifier.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Annotated + +from fastapi import APIRouter, Depends + +from stirling.agents import DocumentClassifierAgent +from stirling.api.dependencies import get_document_classifier_agent +from stirling.contracts import ClassifyDocumentRequest, ClassifyDocumentResponse + +router = APIRouter(prefix="/api/v1/documents/classify", tags=["document-classifier"]) + + +@router.post("", response_model=ClassifyDocumentResponse) +async def classify_document( + request: ClassifyDocumentRequest, + agent: Annotated[DocumentClassifierAgent, Depends(get_document_classifier_agent)], +) -> ClassifyDocumentResponse: + """Classify a document from its supplied page text against the allowed labels. + + The caller sends the bounded page window inline, so no per-user document + storage is touched here β€” the request is self-contained. + """ + return await agent.classify(request) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 77ce40301e..513906d610 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -35,6 +35,12 @@ from .contradiction import ( ContradictionReport, ContradictionSeverity, ) +from .document_classifier import ( + ClassifyDocumentRequest, + ClassifyDocumentResponse, + DocumentClassificationResponse, + LabelOption, +) from .documents import ( DeleteDocumentResponse, IngestDocumentRequest, @@ -128,6 +134,8 @@ __all__ = [ "AiToolAgentStep", "ArtifactKind", "CannotContinueExecutionAction", + "ClassifyDocumentRequest", + "ClassifyDocumentResponse", "Claim", "CommentSpec", "CompletedExecutionAction", @@ -138,6 +146,8 @@ __all__ = [ "DeleteDocumentResponse", "PurgeOwnerResponse", "Discrepancy", + "DocumentClassificationResponse", + "LabelOption", "DocumentMeta", "DocumentSections", "DiscrepancyKind", diff --git a/engine/src/stirling/contracts/document_classifier.py b/engine/src/stirling/contracts/document_classifier.py new file mode 100644 index 0000000000..d64e79f492 --- /dev/null +++ b/engine/src/stirling/contracts/document_classifier.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from pydantic import Field + +from stirling.models import ApiModel + +from .documents import PageText + + +class LabelOption(ApiModel): + """One allowed label: a stable ``id`` (the identity) plus the human ``name`` + the model actually reasons over. The model picks names; the engine maps its + answer back to ids, so the result is language- and rename-stable.""" + + id: str = Field(min_length=1) + name: str = Field(min_length=1) + + +class ClassifyDocumentRequest(ApiModel): + """Classify one document from its page text. + + The caller sends the page text directly β€” typically just the bounded window + (first/last pages), since the classifier reads no more than that. There is no + ingestion or RAG step. + + ``labels`` is the allowed vocabulary for this request β€” the caller (the + backend) always supplies it from the team's stored label set, each an + ``{id, name}`` pair. The engine holds no vocabulary of its own, so at least + one label is required. + """ + + file_name: str = Field(min_length=1) + pages: list[PageText] = Field(default_factory=list) + labels: list[LabelOption] = Field(min_length=1) + + +class DocumentClassificationResponse(ApiModel): + """Terminal classification result. + + ``labels`` is the subset of the allowed vocabulary the model assigned to the + document, as label **ids**: at most five entries, deduplicated, in the + model's order. An empty list is a valid answer β€” nothing in the vocabulary + fit. This is a plain answer from a dedicated endpoint β€” it carries no + ``outcome`` discriminator (it isn't one of the orchestrator's + WorkflowOutcome-routed union responses). + """ + + labels: list[str] = Field(default_factory=list) + + +# Only one response shape today; kept as a named alias so routes and agents have +# a stable response type to import. +ClassifyDocumentResponse = DocumentClassificationResponse diff --git a/engine/tests/test_document_classifier.py b/engine/tests/test_document_classifier.py new file mode 100644 index 0000000000..869d0dc728 --- /dev/null +++ b/engine/tests/test_document_classifier.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from stirling.agents.document_classifier import ( + MAX_ASSIGNED_LABELS, + DocumentClassifierAgent, + _ClassifierOutput, + render_labels, + select_window, + validate_labels, +) +from stirling.contracts import ( + ClassifyDocumentRequest, + DocumentClassificationResponse, + LabelOption, + PageText, +) +from stirling.services.runtime import AppRuntime + + +def _page(number: int, text: str = "x") -> PageText: + return PageText(page_number=number, text=text) + + +def _slug(name: str) -> str: + return re.sub(r"^-+|-+$", "", re.sub(r"[^a-z0-9]+", "-", name.lower())) + + +def _opts(*names: str) -> list[LabelOption]: + """Allowed vocabulary from names, with slug ids (mirrors the frontend).""" + return [LabelOption(id=_slug(name), name=name) for name in names] + + +# ── select_window ─────────────────────────────────────────────────────────── + + +def test_select_window_returns_short_documents_whole() -> None: + pages = [_page(1), _page(2), _page(3), _page(4)] + assert select_window(pages, window=2) == pages + + +def test_select_window_takes_both_ends_without_overlap() -> None: + pages = [_page(n) for n in range(1, 6)] # 5 pages + selected = select_window(pages, window=2) + assert [p.page_number for p in selected] == [1, 2, 4, 5] + + +def test_select_window_handles_empty() -> None: + assert select_window([], window=2) == [] + + +def test_select_window_zero_returns_all() -> None: + pages = [_page(1), _page(2), _page(3)] + assert select_window(pages, window=0) == pages + + +# ── validate_labels ────────────────────────────────────────────────────────── + + +def test_valid_labels_are_preserved_in_order() -> None: + # Model answers with names; the result is the matching label ids. + output = _ClassifierOutput(labels=["Invoice", "Receipt"]) + result = validate_labels(output, _opts("Invoice", "Receipt", "Purchase order")) + assert isinstance(result, DocumentClassificationResponse) + assert result.labels == ["invoice", "receipt"] + + +def test_off_list_labels_are_dropped() -> None: + output = _ClassifierOutput(labels=["Invoice", "Warp core", "Receipt"]) + result = validate_labels(output, _opts("Invoice", "Receipt")) + assert result.labels == ["invoice", "receipt"] + + +def test_matching_is_case_insensitive_and_returns_ids() -> None: + output = _ClassifierOutput(labels=["invoice", " CREDIT NOTE "]) + result = validate_labels(output, _opts("Invoice", "Credit note")) + assert result.labels == ["invoice", "credit-note"] + + +def test_duplicates_collapse_to_first_occurrence() -> None: + output = _ClassifierOutput(labels=["Invoice", "invoice", "Receipt", "INVOICE"]) + result = validate_labels(output, _opts("Invoice", "Receipt")) + assert result.labels == ["invoice", "receipt"] + + +def test_result_is_capped_at_max_assigned_labels() -> None: + allowed = _opts(*[f"Label {n}" for n in range(10)]) + output = _ClassifierOutput(labels=[label.name for label in allowed]) + result = validate_labels(output, allowed) + assert result.labels == [label.id for label in allowed[:MAX_ASSIGNED_LABELS]] + + +def test_empty_answer_is_valid() -> None: + result = validate_labels(_ClassifierOutput(labels=[]), _opts("Invoice")) + assert result.labels == [] + + +def test_entirely_off_list_answer_yields_empty_result() -> None: + output = _ClassifierOutput(labels=["Spaceship", "Boarding pass"]) + result = validate_labels(output, _opts("Invoice", "Receipt")) + assert result.labels == [] + + +# ── render_labels ──────────────────────────────────────────────────────────── + + +def test_render_labels_lists_the_vocabulary_names() -> None: + rendered = render_labels(_opts("Invoice", "Receipt")) + assert "Invoice" in rendered + assert "Receipt" in rendered + + +def test_render_labels_handles_empty_vocabulary() -> None: + assert "(none)" in render_labels([]) + + +# ── DocumentClassifierAgent (inline page text) ─────────────────────────────── + + +def _stub_model_answer(agent: DocumentClassifierAgent, labels: list[str]) -> AsyncMock: + mock = AsyncMock(return_value=SimpleNamespace(output=_ClassifierOutput(labels=labels))) + agent._agent.run = mock + return mock + + +@pytest.mark.anyio +async def test_classify_respects_request_supplied_vocabulary(runtime: AppRuntime) -> None: + agent = DocumentClassifierAgent(runtime) + run_mock = _stub_model_answer(agent, ["board minutes", "Invoice"]) + + result = await agent.classify( + ClassifyDocumentRequest( + file_name="minutes.pdf", + pages=[PageText(page_number=1, text="Minutes of the board meeting")], + labels=_opts("Board minutes", "Agenda"), + ) + ) + + # "Invoice" is off this request's vocabulary even though the default knows it. + assert result.labels == ["board-minutes"] + assert run_mock.await_args is not None + prompt = run_mock.await_args.args[0] + assert "Board minutes" in prompt + assert "Agenda" in prompt diff --git a/engine/tests/test_document_classifier_routes.py b/engine/tests/test_document_classifier_routes.py new file mode 100644 index 0000000000..76715d2862 --- /dev/null +++ b/engine/tests/test_document_classifier_routes.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from fastapi.testclient import TestClient + +from stirling.api import app +from stirling.api.dependencies import get_document_classifier_agent +from stirling.contracts import ( + ClassifyDocumentRequest, + ClassifyDocumentResponse, + DocumentClassificationResponse, +) + + +class StubClassifierAgent: + """Stands in for DocumentClassifierAgent so route tests don't call a model.""" + + def __init__(self, response: ClassifyDocumentResponse) -> None: + self._response = response + + async def classify(self, _request: ClassifyDocumentRequest) -> ClassifyDocumentResponse: + return self._response + + +@pytest.fixture +def classification_client() -> Iterator[TestClient]: + app.dependency_overrides[get_document_classifier_agent] = lambda: StubClassifierAgent( + # The result is label ids (the model's name answers, mapped to ids). + DocumentClassificationResponse(labels=["nda", "contract"]) + ) + try: + yield TestClient(app) + finally: + app.dependency_overrides.pop(get_document_classifier_agent, None) + + +_LABELS = [{"id": "nda", "name": "NDA"}, {"id": "contract", "name": "Contract"}] + + +def test_classify_returns_assigned_labels(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={ + "fileName": "nda.pdf", + "pages": [{"pageNumber": 1, "text": "Mutual NDA between A and B."}], + "labels": _LABELS, + }, + ) + assert response.status_code == 200 + assert response.json() == {"labels": ["nda", "contract"]} + + +def test_classify_accepts_allowed_labels_on_the_request(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={ + "fileName": "nda.pdf", + "pages": [], + "labels": [ + {"id": "contract", "name": "Contract"}, + {"id": "invoice", "name": "Invoice"}, + ], + }, + ) + assert response.status_code == 200 + + +def test_classify_accepts_empty_pages(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={"fileName": "blank.pdf", "pages": [], "labels": _LABELS}, + ) + assert response.status_code == 200 + + +def test_classify_rejects_empty_file_name(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={"fileName": "", "pages": [], "labels": _LABELS}, + ) + assert response.status_code == 422 + + +def test_classify_rejects_missing_labels(classification_client: TestClient) -> None: + # The backend always supplies the vocabulary; a request without one is invalid. + response = classification_client.post( + "/api/v1/documents/classify", + json={"fileName": "nda.pdf", "pages": []}, + ) + assert response.status_code == 422 diff --git a/frontend/.storybook/preview.tsx b/frontend/.storybook/preview.tsx index 76463001fe..5ec17efb9a 100644 --- a/frontend/.storybook/preview.tsx +++ b/frontend/.storybook/preview.tsx @@ -19,11 +19,26 @@ import { UIProvider } from "@portal/contexts/UIContext"; import { SuiProvider } from "@portal/theme/SuiProvider"; import { handlers } from "@portal/mocks/handlers"; import { configureSupabase } from "@proprietary/auth/supabase/supabaseClient"; +import i18next from "i18next"; +import { initReactI18next } from "react-i18next"; import "@mantine/core/styles.css"; import "@core/tokens/tokens.css"; import "@core/tokens/base.css"; +// Storybook-only: init react-i18next so t(key, fallback, vars) interpolates its +// English fallback (there's no backend here to load locale files). Without this, +// the default t() returns raw templates like "{{count}} people Β· led by {{owner}}". +if (!i18next.isInitialized) { + void i18next.use(initReactI18next).init({ + lng: "en", + fallbackLng: "en", + resources: { en: { translation: {} } }, + interpolation: { escapeValue: false }, + react: { useSuspense: false }, + }); +} + // Start MSW once. Storybook runs in a browser so this uses the service worker. initialize({ onUnhandledRequest: "bypass" }, handlers); diff --git a/frontend/editor/.env.proprietary b/frontend/editor/.env.proprietary index 0ee7e66bc9..4d25e380da 100644 --- a/frontend/editor/.env.proprietary +++ b/frontend/editor/.env.proprietary @@ -13,10 +13,12 @@ # VITE_EDITOR_URL=http://localhost:5173/). VITE_EDITOR_URL=/ -# Force the portal's MSW mocks on ("true") or off ("false"). Empty = default -# (on in dev, off in production builds). Set "false" to run against the real -# backend. -VITE_PORTAL_MOCKS= +# Hosted SaaS Supabase project for the self-hosted portal's IN-APP account +# linking (both values are public). Set per deploy; absent -> the account-link +# UI shows a "configure" state. For local e2e, point these at the SaaS Supabase +# project the local backend links against. +VITE_SAAS_SUPABASE_URL= +VITE_SAAS_SUPABASE_ANON_KEY= # Hosted SaaS Java backend base URL (e.g. https://api.stirlingpdf.com). Used for # ATTENDED portal -> SaaS reads (wallet, billing, plans, checkout) with the diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index e81f5eaebb..af5ed16169 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -2812,6 +2812,7 @@ back = "Back" cancel = "Cancel" close = "Close" collapse = "Collapse" +confirm = "Confirm" continue = "Continue" copied = "Copied!" copy = "Copy" @@ -3690,7 +3691,9 @@ workbench = "Workbench" [fileSidebar] addFiles = "Add files" +addingFiles = "Adding files…" collapse = "Collapse sidebar" +customizeGroups = "Customize groups" dropHint = "Open files to get started" dropToAdd = "Drop files to add" expand = "Expand sidebar" @@ -3703,8 +3706,11 @@ noFiles = "No files yet" openFileManager = "Browse all files & folders" openFromComputer = "Open from computer" openSettings = "Open settings" +other = "Other" +recent = "Recent" search = "Search" searchPlaceholder = "Search files..." +viewAll = "View all {{count}} files" [fileSidebar.fileItem] closeViewer = "Close viewer" @@ -3716,6 +3722,23 @@ updateOnServer = "Update on server" uploadToServer = "Upload to server" versionHistory = "Version history" +[fileSidebar.groupsModal] +add = "Add" +addLabel = "Add a label…" +categoryIconAria = "Choose an icon for {{name}}" +createCategory = "Create category" +delete = "Delete category" +done = "Done" +hide = "Hide category" +newCategory = "New category name…" +removeLabel = "Remove {{name}}" +rename = "Rename" +reset = "Reset to defaults" +search = "Search labels…" +show = "Show category" +subtitle = "Group your files into parent categories. Add existing or new labels to a category, rename it, or create your own. Files in none of your categories appear under β€œOther”." +title = "Sidebar categories" + [filesPage] addToWorkspace = "Add to workspace" addToWorkspaceCount = "Add {{count}} to workspace" @@ -3725,6 +3748,7 @@ backToFolder = "Back to {{folder}}" backToMyFiles = "Back to My Files" breadcrumbs = "Folder path" cancel = "Cancel" +classification = "Classification" clearSearch = "Clear search" clearSelection = "Clear selection" closeDetails = "Close details" @@ -3884,10 +3908,10 @@ uploadFilesFailedDetail = "Could not upload files: {{message}}" added = "Added" count = "Files" folder = "Folder" +labels = "Labels" modified = "Modified" name = "Name" size = "Size" -toolHistory = "Tool history" toolHistoryAtVersion = "Cumulative tool chain" totalSize = "Total size" type = "Type" @@ -5956,6 +5980,16 @@ statDocsEnforced = "Docs enforced" statusActive = "Active" statusPaused = "Paused" +[policies.docType] +Contracts = "Contracts" +"Financial reports" = "Financial reports" +"HR records" = "HR records" +Insurance = "Insurance" +Invoices = "Invoices" +"Legal filings" = "Legal filings" +"Medical / PHI" = "Medical / PHI" +"Tax documents" = "Tax documents" + [policies.enforcement] applying = "Applying {{names}}" applyingProgress = "Applying {{names}} ({{done}} of {{total}})" @@ -5977,9 +6011,95 @@ export = "Enforcing before export" input = "Enforcing on import" print = "Enforcing before print" +[policies.field] +accessLog = "Access log" +archiveAfter = "Archive after" +auditTrail = "Audit trail" +belowThreshold = "Below threshold" +destination = "Destination" +frameworks = "Frameworks" +immutableHold = "Immutable hold" +keepFor = "Keep for" +minConfidence = "Min confidence" +notify = "Notify on route" +onViolation = "When non-compliant" +webhookUrl = "Webhook URL" + +[policies.fieldOption.archiveAfter] +"1 year" = "1 year" +"30 days" = "30 days" +"90 days" = "90 days" +Never = "Never" + +[policies.fieldOption.belowThreshold] +"Flag for review" = "Flag for review" +Hold = "Hold" +"Route to bucket" = "Route to bucket" + +[policies.fieldOption.destination] +Documents = "Documents" +"S3 bucket" = "S3 bucket" +SharePoint = "SharePoint" +Webhook = "Webhook" + +[policies.fieldOption.frameworks] +FedRAMP = "FedRAMP" +GDPR = "GDPR" +HIPAA = "HIPAA" +"ISO 27001" = "ISO 27001" +"PCI DSS" = "PCI DSS" +"SOC 2" = "SOC 2" + +[policies.fieldOption.keepFor] +"1 year" = "1 year" +"3 years" = "3 years" +"30 days" = "30 days" +"7 years" = "7 years" +Indefinite = "Indefinite" + +[policies.fieldOption.minConfidence] +"60%" = "60%" +"70%" = "70%" +"80%" = "80%" +"90%" = "90%" +"95%" = "95%" + +[policies.fieldOption.onViolation] +"Auto-redact PHI" = "Auto-redact PHI" +"Block export" = "Block export" +"Flag for review" = "Flag for review" +"Quarantine document" = "Quarantine document" + [policies.fields] selectedCount = "{{count}} selected" +[policies.labels] +add = "Add" +addPlaceholder = "Add a label…" +customNote = "Customized for your team." +defaultNote = "Using the built-in default, shared with your team." +duplicate = "\"{{name}}\" already exists." +edit = "Edit labels" +empty = "No labels yet." +export = "Export JSON" +iconAria = "Choose an icon for {{name}}" +import = "Import JSON" +importError = "Couldn't import that file." +managedNote = "Team labels are managed by your team leader." +modalSubtitle = "Shared with your whole team. The classifier picks the labels that fit each document." +modalTitle = "Classification labels" +pickIcon = "Use {{label}} icon" +removeAria = "Remove {{name}}" +resetToDefault = "Reset to default" +saveForTeam = "Save for team" +saving = "Saving…" +sectionLabel = "Classification labels" +startFromScratch = "Start from scratch" +teamCount = "team labels" +tooLong = "Labels can be at most {{max}} characters." +ungrouped = "Ungrouped" +view = "View labels" + [policies.pii] account = "Account numbers (labelled)" card = "Credit / debit cards" @@ -5991,17 +6111,30 @@ placeholder = "Select PII types" routing = "US routing numbers (ABA)" ssn = "Social Security numbers" +[policies.settings] +noneExport = "No policies currently run on export." +noneUpload = "No policies currently run on upload." +onExport = "On export" +onUpload = "On upload" +reorderHandle = "Drag to reorder" +runOrderDesc = "When more than one policy runs on the same trigger, they run in this order β€” each on the previous policy's output. Drag to reorder." +title = "Policy settings" + [policies.sidebar] activeCount = "{{count}} active" -infoAriaLabel = "What is a policy?" infoTooltip = "A policy is a fixed set of tools that runs automatically whenever it's triggered β€” for example when a new document arrives β€” enforcing rules like redacting PII with no manual steps." loading = "Loading…" +optionsAriaLabel = "Policy options" +policySettings = "Policy settings" railAriaLabel = "{{label}} policy β€” {{status}}" railSuffixActive = " (Active)" railSuffixPaused = " (Paused)" +retryFailed = "Retry failed policies ({{count}})" +rowProgress = "{{completed}} of {{total}} files processed" setUp = "Set up" title = "Policies" upgradeToEnterprise = "Upgrade to enterprise" +whatIsPolicy = "What is a policy?" [policies.status] active = "Active" @@ -6462,8 +6595,18 @@ componentSpendMtd = "Component spend (MTD)" embedsThisMonth = "Embeds this month" inBeta = "In beta" -[portal.common] -inviteMember = "Invite member" +[portal.components.billingUnit] +approval = "approval" +check = "check" +event = "event" +render = "render" +review = "review" +session = "session" +signature = "signature" + +[portal.components.maturity] +beta = "Beta" +ga = "GA" [portal.componentsView] subtitle = "Embeddable SDK widgets you drop into your own app β€” a viewer, an e-sign flow, an AI review panel. Each is metered per action. Click a card for install, usage and props." @@ -6585,7 +6728,14 @@ subtitle = "Every document your org has processed, with its full processing reco title = "Documents" [portal.documents.audit] +approved = "Approved" +archived = "Archived" +elevation = "Elevation" empty = "No events recorded yet." +extracted = "Processed" +flagged = "Needs Review" +ingested = "Ingested" +reviewed = "In Review" [portal.documents.drawer] sectionsAriaLabel = "Document detail sections" @@ -6636,6 +6786,12 @@ user = "User" description = "As sources feed documents into your pipelines they'll appear here for review." title = "No documents in the queue" +[portal.documents.status] +error = "Error" +flagged = "Needs Review" +inReview = "In Review" +processed = "Processed" + [portal.documents.table] auto = "Auto" editorAction = "Editor" @@ -6710,6 +6866,12 @@ title = "Service token" description = "A new token was issued. Update each self-hosted instance's STIRLING_SERVICE_TOKEN within the 24h grace window or they'll drop offline." title = "Rotate running instances" +[portal.editorAdmin.status] +degraded = "Degraded" +healthy = "Healthy" +offline = "Offline" +pairing = "Pairing" + [portal.editorAdmin.targets] instanceCount_one = "{{count}} instance" instanceCount_other = "{{count}} instances" @@ -6730,23 +6892,6 @@ description = "This view hit an unexpected error. Try again, or pick another sec retry = "Try again" title = "Something went wrong on this page" -[portal.forkWizard] -subtitle = "Clone a proven workflow and tune it β€” every template ships the same four-stage backbone." -title = "Fork a starter pipeline" - -[portal.forkWizard.action] -cancel = "Cancel" -deploy = "Deploy pipeline" -pickAnother = "Pick another" - -[portal.forkWizard.status] -building = "Building…" -ready = "Ready to deploy" - -[portal.home.chart.empty] -description = "Once documents are processed, your 30-day usage appears here." -title = "No usage yet" - [portal.home.editor] activeUsers = "{{n}} active" invite = "Invite teammates" @@ -6764,25 +6909,6 @@ afternoon = "Good afternoon" evening = "Good evening" morning = "Good morning" -[portal.home.kpis.enterprise] -docs30d = "Docs / 30d" -evalPassRate = "Eval pass rate" -p95Latency = "P95 latency" -slaUptime = "SLA uptime (30d)" - -[portal.home.kpis.free] -agents = "Agents" -docsProcessed = "Docs processed" -docsProcessedDescription = "Free plan cap" -operations = "Operations" -pipelines = "Pipelines" - -[portal.home.kpis.pro] -agentsActive = "Agents active" -docs30d = "Docs / 30d" -evalPassRate = "Eval pass rate" -pipelines = "Pipelines" - [portal.home.onboarding] dismiss = "Dismiss setup" notStarted = "Not started" @@ -6810,25 +6936,6 @@ blurb = "{{connected}} connected Β· every PDF governed where it lands" chip = "{{connected}} connected" title = "Connect your sources" -[portal.home.productGrid] -ariaLabel = "Process PDFs at scale" - -[portal.home.productGrid.agents] -blurb = "Wire your agent via MCP, REST, or tool definitions. Deterministic operations, scenarios, evals." -cta = "Connect an agent" -title = "Agents" - -[portal.home.productGrid.pipelines] -badge = "Hero" -blurb = "Compose document workflows from typed operations. Upload a sample to get suggestions or start blank." -cta = "Build a pipeline" -title = "Pipelines" - -[portal.home.productGrid.sources] -blurb = "Attach pipelines where PDFs already live β€” S3, agents, SharePoint, webhooks, batch, email." -cta = "Connect a source" -title = "Sources" - [portal.home.quickActions] subtitle = "Top tasks for today" title = "Quick actions" @@ -6845,18 +6952,6 @@ title = "Connect a source" blurb = "Scoped key with rate limits and IP allowlist" title = "Issue an API key" -[portal.home.quickActions.tryOp] -blurb = "Drop a sample, pick an op, see the JSON" -title = "Try a PDF operation" - -[portal.home.regions] -subtitle = "Real-time status for every deployed Stirling region." -title = "Region health" - -[portal.home.regions.empty] -description = "Once a region is deployed, its health appears here." -title = "No regions yet" - [portal.infrastructure] manageEditorDeployment = "Manage Editor deployment" sectionsAriaLabel = "Infrastructure sections" @@ -7165,14 +7260,6 @@ models = "Models" security = "Security" storage = "Storage" -[portal.mocks.label] -off = "Mocks OFF" -on = "Mocks ON" - -[portal.mocks.tooltip] -off = "Mock data OFF β€” fetch calls go to the real network. Click to re-enable mocks (reloads the page)." -on = "Mock data ON β€” fetch calls are intercepted by MSW. Click to switch to the real network (reloads the page)." - [portal.nav] agent-builder = "Agent Builder" components = "Components" @@ -7209,51 +7296,6 @@ new_other = "{{count}} new" description = "No new notifications." title = "You're all caught up" -[portal.opRunner] -durationMs = "{{ms}} ms" -featuredOps = "Featured ops" -subtitle = "Drop a sample, pick an op, see what Stirling returns." -title = "Try a PDF operation" - -[portal.opRunner.action] -close = "Close" -openBuilder = "Open the pipeline builder" -run = "Run operation" -runAgain = "Run again" -running = "Running…" - -[portal.opRunner.drop] -hint = "or use a sample document." -pickAnother = "Pick another sample" -replaceHint = "Drop again or pick another sample to replace." -title = "Drop a PDF here" -useSample = "Use a sample" - -[portal.opRunner.empty] -description = "Once operations are published, they'll show up here." -title = "No featured ops yet" - -[portal.opRunner.error] -title = "The operation didn't complete" -unknown = "Unknown error" - -[portal.opRunner.hint] -aSample = "a sample" -press = "Press" -ready = "Ready" -toInvoke = "to invoke" - -[portal.opRunner.hint.runOn] -before = "Run" -middle = "on" - -[portal.opRunner.running] -title = "Running {{label}}…" - -[portal.opRunner.status] -completed = "Completed" -failed = "Failed" - [portal.pipelines] subtitle = "Every automated document pipeline on the backend: an ordered chain of operations over a set of sources, run on a trigger. Click a row for its steps and sources." title = "Pipelines" @@ -7362,9 +7404,93 @@ subtitle = "Standing automations that enforce a tool pipeline on every document. title = "Policies" [portal.policies.card] -comingSoon = "Coming soon" +comingSoon = "Upgrade to Enterprise" notSetUp = "Not set up" +[portal.policies.categories.compliance] +desc = "Enforce HIPAA, GDPR, SOC 2, or FedRAMP requirements on every document." +label = "Compliance" + +[portal.policies.categories.ingestion] +desc = "Classify documents, extract structured data, enforce naming conventions, and normalize pages." +label = "Ingestion" + +[portal.policies.categories.retention] +desc = "Set how long documents are kept, when to archive, and when to delete." +label = "Retention" + +[portal.policies.categories.routing] +desc = "Auto-route documents to the right team, folder, or system." +label = "Routing" + +[portal.policies.categories.security] +desc = "Detect PII, redact, strip active content, and watermark documents." +label = "Security" + +[portal.policies.config] +scopeAll = "All documents" + +[portal.policies.config.compliance] +summary = "Validates documents against regulatory frameworks before they leave the system." + +[portal.policies.config.compliance.fields] +accessLog = "Access log" +auditTrail = "Audit trail" +frameworks = "Frameworks" +onViolation = "When non-compliant" + +[portal.policies.config.compliance.rules] +0 = "Framework scan" +1 = "Enforce action" +2 = "Audit trail" + +[portal.policies.config.ingestion] +summary = "Classifies documents, extracts structured data, enforces naming, and normalizes pages." + +[portal.policies.config.ingestion.fields] +belowThreshold = "Below threshold" +minConfidence = "Min confidence" + +[portal.policies.config.ingestion.rules] +0 = "Classify" +1 = "Extract" +2 = "Name" +3 = "Normalize" + +[portal.policies.config.retention] +summary = "Enforces how long documents are kept, when to archive, and when to delete." + +[portal.policies.config.retention.fields] +archiveAfter = "Archive after" +immutableHold = "Immutable hold" +keepFor = "Keep for" + +[portal.policies.config.retention.rules] +0 = "Retention hold" +1 = "Auto-archive" +2 = "Deletion block" + +[portal.policies.config.routing] +summary = "Routes documents to the right destination based on type and classification." + +[portal.policies.config.routing.fields] +destination = "Destination" +notify = "Notify on route" +webhookUrl = "Webhook URL" + +[portal.policies.config.routing.rules] +0 = "Auto-classify" +1 = "Route to folder" +2 = "Webhook notify" + +[portal.policies.config.security] +summary = "Detects and redacts PII, strips active content (JavaScript), and watermarks documents." + +[portal.policies.config.security.rules] +0 = "Redact PII" +1 = "Remove JavaScript" +2 = "Watermark" + [portal.policies.detail] enforces = "Enforces" onEveryExport = "On every export" @@ -7378,16 +7504,31 @@ showMore = "Show more" sources = "Sources" [portal.policies.detail.actions] +clearHistory = "Clear processed history" delete = "Delete" editSettings = "Edit settings" pause = "Pause" resume = "Resume" runNow = "Run now" +[portal.policies.detail.clearHistory] +body = "This policy will forget every file it has already processed and reprocess everything currently in its sources on the next run. The files themselves are not changed. This cannot be undone." +cancel = "Cancel" +confirm = "Clear history" +title = "Clear processed history?" + [portal.policies.detail.emptyActivity] description = "Documents will appear here once this policy runs." title = "No activity yet" +[portal.policies.endpoints] +addWatermark = "Watermark" +autoRedact = "Redact PII" +compressPdf = "Compress" +flatten = "Flatten" +ocrPdf = "OCR" +sanitizePdf = "Remove JavaScript" + [portal.policies.offline] description = "Your policies are saved and will appear once the connection is restored." retry = "Retry" @@ -7425,14 +7566,29 @@ continue = "Continue" enablePolicy = "Enable policy" saveChanges = "Save changes" -[portal.policies.wizard.docTypes] -allDescription = "Set up an Ingestion (classification) policy to narrow this to specific document types." -allTitle = "All document types" -clear = "Clear" -heading = "Document types" -narrow = "Narrow" -selected_one = "{{count}} selected" -selected_other = "{{count}} selected" +[portal.policies.wizard.capability.compress] +desc = "Compresses the document to a smaller file size." +label = "Reduce file size" + +[portal.policies.wizard.capability.flatten] +desc = "Merges form fields and annotations into the page so they can't be edited." +label = "Flatten the document" + +[portal.policies.wizard.capability.ocr] +desc = "Runs OCR so scanned pages become selectable, searchable text." +label = "Make text searchable" + +[portal.policies.wizard.capability.redact] +desc = "Finds and blacks out sensitive details β€” like Social Security and card numbers β€” so they can't be read." +label = "Redact sensitive information" + +[portal.policies.wizard.capability.sanitize] +desc = "Removes hidden JavaScript so nothing can run automatically when the document is opened." +label = "Strip active content" + +[portal.policies.wizard.capability.watermark] +desc = "Stamps a visible mark (e.g. β€œConfidential”) across every page." +label = "Apply a watermark" [portal.policies.wizard.errors] noTools = "Enable at least one tool in the workflow first." @@ -7453,11 +7609,6 @@ label = "Output as" newFile = "New file" newVersion = "New version" -[portal.policies.wizard.output.retries] -delayLabel = "Retry delay (min)" -heading = "Retries" -maxLabel = "Max retries" - [portal.policies.wizard.output.runOn] export = "Export" helper = "When the policy fires: on upload, or before export." @@ -7468,22 +7619,20 @@ upload = "Upload" heading = "Settings" [portal.policies.wizard.sources] -emptyDescription = "Connect a source on the Sources page first, then attach it to a policy here." -emptyTitle = "No sources available" heading = "Sources" loading = "Loading sources…" [portal.policies.wizard.tabs] ariaLabel = "Setup steps" settings = "Settings" -workflow = "Workflow" +workflow = "Actions" [portal.policies.wizard.title] edit = "Edit {{category}} policy" setUp = "Set up {{category}} policy" [portal.policies.wizard.workflow] -description = "The sequence of tools this policy runs on each document. Each tool is a Stirling endpoint; toggle the ones this policy should enforce." +description = "Choose what this policy does to every document it processes." [portal.policySummary] activeSummary = "{{active}} / {{total}} active" @@ -7492,7 +7641,7 @@ subtitle = "Standing automations every document passes through, regardless of wh title = "What runs on your PDFs" [portal.policySummary.action] -comingSoon = "Coming soon" +comingSoon = "Upgrade to Enterprise" configure = "Configure" setUp = "Set up" @@ -7512,9 +7661,6 @@ soon = "Soon" [portal.processingStatus] managePlan = "Manage plan" -pdfsThisMonth = "PDFs this month" -progressLabel = "{{used}} of {{cap}} PDFs used this month" -upgrade = "Upgrade" volumeSuffix = "PDFs processed Β· last 30 days" [portal.procurement] @@ -7638,6 +7784,31 @@ subtitle = "Your solutions engineer is on every step. One next action at a time; title = "From trial to live, one guided path" trialTitle = "Enterprise trial" +[portal.procurement.journeySteps.agreement] +blurb = "One signature covers MSA, order form, EULA and DPA." +gatingAction = "Review and sign your agreement" +label = "Agreement" + +[portal.procurement.journeySteps.implementation] +blurb = "Provision your workspace and run the go-live playbook." +gatingAction = "Provisioning your workspace" +label = "Implementation" + +[portal.procurement.journeySteps.payment] +blurb = "Pay by card, bank transfer, or against a purchase order." +gatingAction = "Confirm payment" +label = "Payment" + +[portal.procurement.journeySteps.quote] +blurb = "Review committed-volume pricing and contract term." +gatingAction = "Accept your quote" +label = "Quote" + +[portal.procurement.journeySteps.trial] +blurb = "Evaluate Stirling against your documents and workflows." +gatingAction = "Build your quote" +label = "Trial" + [portal.procurement.license] copied = "Copied" copy = "Copy key" @@ -7750,122 +7921,11 @@ noActionsTitle = "No quick actions" noMatches = "No matches for \"{{query}}\"" noMatchesDescription = "Try a different keyword or browse the catalogue." -[portal.settings] -ariaLabel = "Settings" -cancel = "Cancel" -enterpriseBadge = "Enterprise" -footerNote = "Changes apply to this workspace." -saveChanges = "Save changes" - -[portal.settings.appearance] -themeSub = "Choose how the portal looks on this device." -themeTitle = "Theme" - -[portal.settings.appearance.dark] -hint = "Dim surfaces" -label = "Dark" - -[portal.settings.appearance.light] -hint = "Bright surfaces" -label = "Light" - -[portal.settings.authentication] -sessionTimeout = "Session timeout" -sessionTimeoutHelper = "Members re-authenticate after this idle period." -sub = "Organisation-wide authentication controls." -title = "Sign-in policy" - -[portal.settings.authentication.mfa] -description = "Require every member to complete MFA at sign-in." -label = "Enforce two-factor (MFA)" - -[portal.settings.authentication.scim] -description = "Sync members and roles from your directory." -label = "SCIM provisioning" - -[portal.settings.authentication.sso] -description = "Federate sign-in through your identity provider." -label = "Single sign-on (SAML)" - -[portal.settings.authentication.timeout] -1440 = "24 hours" -240 = "4 hours" -480 = "8 hours" -60 = "1 hour" -720 = "12 hours" - -[portal.settings.earlyAccess] -sub = "Opt into features still in preview." -title = "Preview features" - [portal.settings.groups] -account = "Account" admin = "Admin" -workspace = "Workspace" - -[portal.settings.notifications] -sub = "Pick which events reach your inbox." -title = "Email notifications" - -[portal.settings.notifications.pipeline-failures] -description = "A run errors out or a step times out." -label = "Pipeline failures" - -[portal.settings.notifications.pipeline-success] -description = "Every successful pipeline run finishes." -label = "Pipeline completions" - -[portal.settings.notifications.product-updates] -description = "New operations, sources, and release notes." -label = "Product updates" - -[portal.settings.notifications.security-alerts] -description = "New API keys, sign-ins, or permission changes." -label = "Security alerts" - -[portal.settings.notifications.usage-alerts] -description = "You approach a plan limit or rate cap." -label = "Usage & quota alerts" - -[portal.settings.notifications.weekly-digest] -description = "A Monday summary of volume and health." -label = "Weekly digest" - -[portal.settings.profile] -accountFallback = "Account" -changePhoto = "Change photo" -email = "Email" -emailHelper = "Used for sign-in and notification delivery." -emailPlaceholder = "you@company.com" -fullName = "Full name" -namePlaceholder = "Your name" [portal.settings.sections] account-link = "Account link" -appearance = "Appearance" -authentication = "Authentication" -early-access = "Early access" -general = "General" -notifications = "Notifications" -profile = "Profile" -sessions = "Active sessions" - -[portal.settings.sessions] -revoke = "Revoke" -sub = "Devices currently signed in to this account." -thisDevice = "This device" -title = "Active sessions" - -[portal.settings.workspace] -manageBilling = "Manage billing" -nameLabel = "Workspace name" -namePlaceholder = "Workspace name" -plan = "Plan" -regionEnterpriseSuffix = "{{region}} Β· Enterprise" -regionHelper = "Where documents are processed and stored at rest." -regionLabel = "Data residency region" -seats = "Seats" -seatsUsed = "{{used}} of {{total}} used" [portal.shell.header] accountFallback = "Account" @@ -7883,8 +7943,8 @@ appEditor = "Editor" appProcessor = "Processor" brandSuffix = "Stirling Processor" docsCount = "{{docs}} docs" -docsProcessed = "Docs processed" linkAccount = "Link Stirling account" +planEditor = "Editor plan" planEnterprise = "Enterprise plan" planProcessor = "Processor plan" primaryNav = "Primary navigation" @@ -7950,6 +8010,14 @@ helperText = "Absolute path Stirling watches for files to process." label = "Directory path" placeholder = "/data/incoming" +[portal.sources.types.folder.fields.identity] +helperText = "Content check reads each changed file, so renames and touches that don't alter content are not reprocessed." +label = "Change detection" + +[portal.sources.types.folder.fields.identity.options] +hash = "Size, date and content check" +stat = "Size and date modified" + [portal.sources.types.folder.fields.mode] label = "Read mode" @@ -7957,6 +8025,13 @@ label = "Read mode" consume = "Consume: process each file once" snapshot = "Snapshot: re-read the folder every run" +[portal.sources.types.folder.fields.recursive] +label = "Folder depth" + +[portal.sources.types.folder.fields.recursive.options] +all = "Include subfolders" +top = "Top level only" + [portal.sources.types.unknown] label = "Source" @@ -7992,123 +8067,59 @@ action = "Sign in again" body = "Your Stirling account session has expired. Sign in again to view billing β€” your instance stays linked." title = "Session expired" -[portal.usageChart] -defaultLabel = "Docs processed Β· last 30 days" -delta = "{{pct}}% vs prior 30d" -docsValue = "{{value}} docs" -srAnnounce = "{{date}}: {{value}} docs" - -[portal.useCases] -title = "Popular use cases" -viewAll = "View all pipelines" - -[portal.useCases.items.authenticity] -blurb = "Cryptographic checks at the document boundary β€” signature validation, tamper detection, signing flows for outbound documents. Trust decisions in the pipeline, not your app code." -cta = "Try authenticity check" -eyebrow = "AUTHENTICITY" -title = "Verify signatures and detect tampering" - -[portal.useCases.items.autoRouting] -blurb = "One classifier reads what arrived β€” KYC form, invoice, contract, COI β€” and routes to the right downstream pipeline. No manual triage, no docs in the wrong workflow." -cta = "Build a classifier pipeline" -eyebrow = "AUTO-ROUTING" -title = "Auto-classify and route incoming documents" - -[portal.useCases.items.piiRedaction] -blurb = "Strip sensitive fields before storage, indexing, or LLM processing. Schema-aware, per-field audit, BYOK or HYOK keys. Compliance at the document boundary, not per pipeline." -cta = "See redaction pipelines" -eyebrow = "PII REDACTION" -title = "Redact PII before it leaves your stack" - -[portal.useCases.items.trainingData] -blurb = "Batch-import an archive, redact PII, classify, chunk, and emit ready-to-load JSON for fine-tuning, eval sets, or RAG. Self-completing and replayable." -cta = "Build a training-data pipeline" -eyebrow = "TRAINING DATA" -title = "Turn PDFs into training data" - [portal.users] -subtitle = "The people in your organization and the access they hold β€” roles, status and security controls." title = "Users" -[portal.users.access] -subtitle = "Seats, authentication and provisioning for your organization." -title = "Access & security" - -[portal.users.access.auth] -title = "Authentication" - -[portal.users.access.auth.requireMfa] -description = "Members must set up a second factor to sign in." -enforced = "Enforced org-wide on this plan." -label = "Require MFA" - -[portal.users.access.auth.shortSessions] -description = "Sign members out after inactivity (currently {{timeout}})." -label = "Short-lived sessions" - -[portal.users.access.scim] -active = "Active" -directory = "Directory" -lastSync = "Last sync" -note = "Members are created, updated and deactivated automatically from your identity provider." -off = "Off" -title = "SCIM provisioning" - -[portal.users.access.seats] -title = "Seats" -unlimited = "Your plan includes unlimited seats." -usedLabel = "{{used}} of {{limit}} seats used" - -[portal.users.access.sso] -connected = "Connected" -domains = "Domains" -manage = "Manage connection" -notConfigured = "Not configured" -provider = "Provider" -title = "SSO / SAML" - -[portal.users.access.upgrade] -action = "Upgrade plan" -title = "Unlock team access controls" - -[portal.users.empty] -description = "Invite your team to start collaborating on documents and pipelines." -title = "No members yet" - -[portal.users.invite] -cancel = "Cancel" -email = "Email" -emailError = "Enter a valid email address" -emailPlaceholder = "teammate@acme.com" -role = "Role" -roleHelper = "Determines what the member can do once they join." -send = "Send invite" -subtitle = "They'll receive an email to join your organization." - [portal.users.roles] subtitle = "Every role exists on every plan β€” what each one can do is fixed across the org." title = "Roles" -[portal.users.summary] -members = "Members" -pendingInvites = "Pending invites" -seatsUsed = "Seats used" +[portal.users.roles.admin] +label = "Admin (Org owner)" +summary = "Full governance over the workspace, settings and members." -[portal.users.table] -actionsFor = "Actions for {{name}}" -changeRole = "Change role" -lastActive = "Last active" -member = "Member" -remove = "Remove from org" -role = "Role" -status = "Status" -suspend = "Suspend" +[portal.users.roles.admin.permissions] +0 = "Manage users, teams and roles" +1 = "Manage all integrations incl. S3 connections" +2 = "Grant or revoke portal access" +3 = "Everything Team Owner can do" + +[portal.users.roles.guest] +label = "Guest" +summary = "Limited or web-only access; cannot hold personal configs." + +[portal.users.roles.guest.permissions] +0 = "Web-only / demo usage" +1 = "No API keys or integrations" +2 = "No portal access" +3 = "Read-only where shared" + +[portal.users.roles.member] +label = "Member" +summary = "Regular user β€” works with shared resources and their own configs." + +[portal.users.roles.member.permissions] +0 = "Use the editor and shared integrations" +1 = "Create personal API & MCP configs" +2 = "See team configs shared with them" +3 = "No S3 or workspace management" + +[portal.users.roles.team_owner] +label = "Team owner" +summary = "Owns a team β€” manages its members' resources and shared configs." + +[portal.users.roles.team_owner.permissions] +0 = "Create & manage the team's S3 connections" +1 = "Manage team-owned integration configs" +2 = "Portal access via the default policy" +3 = "Everything Member can do" [portal.welcome] ariaLabel = "Welcome to Stirling PDF" badge = "Open-source" installEditor = "Install the Editor" inviteTeammates = "Invite teammates" +openInBrowser = "Open in browser" perks = "Free forever Β· Self-hostable" subtitle = "The world's most secure PDF Editor is free for teams of all sizes. Includes 60+ PDF operations and SSO." title = "Welcome to" @@ -10017,6 +10028,146 @@ title = "Detailed Statistics" unknownEndpoint = "Unknown endpoint" visits = "Visits" +[users] +lastActive = "Last active" +learnMore = "Learn more about roles and access." +locked = "Locked" +roleFor = "Role for {{name}}" +rowActions = "Actions for {{name}}" +showAll = "Show all {{count}}" +showLess = "Show less" +subtitle2 = "Your people, teams, and access levels." +suspended = "Suspended" +teamActions = "Team actions" +title = "Users" +you = "(you)" + +[users.action] +deleteTeam = "Delete team" +disableMfa = "Reset MFA" +move = "Move to team" +reinstate = "Reinstate" +remove = "Remove from org" +rename = "Rename team" +resetPw = "Reset password" +suspend = "Suspend" +unlock = "Unlock account" + +[users.cap] +addProcessor = "+ Processor" +approver = "Approves policy" +editor = "Editor" +processor = "Processor" + +[users.confirm] +deleteTeamBody = "Delete the {{name}} team? The team must be empty first - move its members to another team, and it can't still own any integration configs." +deleteTeamTitle = "Delete team" +disableMfaBody = "Remove {{name}}'s MFA enrolment? They'll set it up again on next login if required." +disableMfaTitle = "Reset MFA" +removeBody = "Permanently remove {{name}} from the organization? This cannot be undone." +removeTitle = "Remove member" + +[users.empty] +description = "Invite your team to start collaborating." +title = "No members yet" + +[users.group] +addToTeam = "Add to team" +guestCount = "{{count}} guest" +guests = "Guests" +guestsDesc = "External collaborators, scoped to what you shared. Editor only." +ledBy = "led by {{owner}}" +org = "Organization" +orgDesc = "Owners with org-wide authority and policy approval" +owners = "{{count}} owner" +team = "{{name}} team" +teamMeta = "{{count}} people" + +[users.invite] +access = "Access" +action = "Invite people" +authOauth = "OAuth2 / SSO" +authSaml = "SAML 2.0" +authType = "Sign-in method" +authWeb = "Password" +create = "Create account" +createSubtitle = "Create a self-hosted account with a password or SSO." +createTitle = "Create account" +editorDesc = "Edit PDFs in the Stirling PDF Editor. Everyone gets this." +email = "Email address" +emailError = "Enter a valid email address" +emailPlaceholder2 = "name@company.com" +forceChange = "Require a password change on first login" +forceMfa = "Require MFA setup on first login" +method = "How to add them" +methodDirect = "Create account directly" +methodEmail = "Invite by email" +password = "Password" +passwordError = "Password must be at least 8 characters" +processorDeferred = "Invite sent, but Processor access couldn't be granted yet - set it from the roster once they've joined." +processorDesc = "The governance surface, run pipelines, agents, and the API." +role = "Role" +send2 = "Send invite" +subtitle2 = "They'll get an email to join your Stirling workspace." +team = "Team" +title = "Invite people" +username = "Username" +usernameError = "Username must be at least 3 characters" +usernamePlaceholder = "jsmith" + +[users.loadError] +description = "Something went wrong reaching the backend, or you don't have access. Try again." +title = "Couldn't load members" + +[users.moveTeam] +apply = "Move" +team = "Team" +title = "Move to team" + +[users.newTeam] +action = "+ New team" +create = "Create team" +emailError = "Enter a valid email address" +name = "Team name" +namePlaceholder = "e.g. Finance" +nameRequired = "Team name is required" +owner = "Team Owner" +ownerFailed = "Team created, but the owner couldn't be invited. Assign one from the roster." +ownerHelper = "Every team has a leader. They'll be invited as Team Owner, with the Processor on, and can add the rest. You can reassign later." +ownerPlaceholder = "owner@company.com" +subtitle = "Group people under a Team Owner who manages their access." +title = "New team" + +[users.renameTeam] +apply = "Rename" +name = "Team name" +title = "Rename team" + +[users.resetPw] +apply = "Reset password" +confirm = "Confirm password" +copied = "Copied to clipboard" +copyHint = "Copy this now - it won't be shown again." +email = "Email the user about the reset" +forceChange = "Require a password change on next login" +generate = "Generate a secure password" +includePw = "Include the new password in the email" +mismatch = "Passwords do not match" +newPassword = "New password" +regen = "Regenerate" +title = "Reset password" +tooShort = "Password must be at least 8 characters" + +[users.role] +guest = "Guest" +member = "Member" +orgOwner = "Org Owner" +teamOwner = "Team Owner" + +[users.team] +grantProcessor = "Grant Processor to team" +revokeProcessor = "Revoke Processor from team" + [validateSignature] date = "Date" downloadCsv = "Download CSV" diff --git a/frontend/editor/public/og-metadata.json b/frontend/editor/public/og-metadata.json index 65f1da789c..1fc552cd80 100644 --- a/frontend/editor/public/og-metadata.json +++ b/frontend/editor/public/og-metadata.json @@ -499,6 +499,11 @@ "image": "/og_images/home.png", "title": "Payg Settings - Stirling PDF", "description": "The Free Adobe Acrobat alternative (10M+ Downloads)" + }, + "/settings/account-link": { + "image": "/og_images/home.png", + "title": "Account Link Settings - Stirling PDF", + "description": "The Free Adobe Acrobat alternative (10M+ Downloads)" } }, "byPath": { @@ -653,6 +658,7 @@ "/settings/legal": "/settings/legal", "/settings/backendThirdPartyLicenses": "/settings/backendThirdPartyLicenses", "/settings/frontendThirdPartyLicenses": "/settings/frontendThirdPartyLicenses", - "/settings/payg": "/settings/payg" + "/settings/payg": "/settings/payg", + "/settings/account-link": "/settings/account-link" } } diff --git a/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx b/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx index fe834b43d7..e183e45d6e 100644 --- a/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx +++ b/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx @@ -295,6 +295,9 @@ const FileEditorThumbnail = ({ const [showVersionHistory, setShowVersionHistory] = useState(false); const policyEnforcing = policies.some((p) => p.enforcing); + // Accent of the policy currently enforcing, so the overlay's icon/spinner match + // that policy's badge instead of a fixed blue. + const enforcingAccent = policies.find((p) => p.enforcing)?.accentColor; const hoverActions = useMemo(() => { const uploadLabel = isUploaded @@ -536,7 +539,11 @@ const FileEditorThumbnail = ({ )} {/* Policy enforcement overlay β€” shown while any policy is in-flight */} - + {/* Thumbnail image or loading state */} (null); + const [classificationOpen, setClassificationOpen] = useState(false); + const classificationEnabled = useClassificationEnabled(); + const labelName = useLabelName(); // Version chain for the selected file; empty for v1 or multi-select. const [versionChain, setVersionChain] = useState([]); const singleFileForChain = files.length === 1 ? files[0] : null; @@ -103,6 +113,29 @@ export function FileDetailsPanel({ }; }, [singleFileForChain]); + // Show the file's classification labels: the stub's cached copy when present + // (free β€” no byte load), else read the PDF metadata via the shared service. + // Gated on classification being enabled (SaaS + AI): off-feature we never read + // the metadata or show the section, so a PDF that happens to carry the + // StirlingPDFClassification key never reveals the feature in a build without it. + useEffect(() => { + setClassification(null); + if (!classificationEnabled) return; + const stub = singleFileForChain; + if (!stub) return; + if (stub.classificationLabels && stub.classificationLabels.length > 0) { + setClassification(stub.classificationLabels); + return; + } + let cancelled = false; + void readStubClassificationLabels(stub).then((labels) => { + if (!cancelled && labels) setClassification(labels); + }); + return () => { + cancelled = true; + }; + }, [singleFileForChain, classificationEnabled]); + if (files.length === 0) { return null; } @@ -241,17 +274,56 @@ export function FileDetailsPanel({ /> )} - {single.toolHistory && single.toolHistory.length > 0 && ( -

-
- {t("filesPage.field.toolHistory", "Tool history")} -
- -
+ {classification && ( + <> + + {classificationOpen && ( +
+
+ + {t("filesPage.field.labels", "Labels")} + + + {classification.map((label) => ( + + {labelName(label)} + + ))} + +
+
+ )} + )} {/* Version journey. Each tool run writes a new StirlingFile with the same `originalFileId` and an incremented @@ -275,12 +347,41 @@ export function FileDetailsPanel({ )} ) : ( - + <> + + {versionsOpen && ( + + )} + ))} ) : ( diff --git a/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx b/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx index 0b6a75878b..47495fe61b 100644 --- a/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx +++ b/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx @@ -57,6 +57,7 @@ export interface VersionTimelineProps { currentId: FileId; onAddToWorkspace: (fileIds: FileId[]) => void; onRemove: (fileIds: FileId[]) => void; + hideHeader?: boolean; } /** Version timeline with per-row tool deltas and collapse-when-long. */ @@ -65,6 +66,7 @@ export function VersionTimeline({ currentId, onAddToWorkspace, onRemove, + hideHeader = false, }: VersionTimelineProps) { const { t } = useTranslation(); const [expandedIds, setExpandedIds] = useState>(new Set()); @@ -122,15 +124,17 @@ export function VersionTimeline({ return (
-
- - {t("filesPage.field.versionHistory", "Version journey")} - - {t("filesPage.versionsCount", "{{count}} versions", { - count: ordered.length, - })} - -
+ {!hideHeader && ( +
+ + {t("filesPage.field.versionHistory", "Version journey")} + + {t("filesPage.versionsCount", "{{count}} versions", { + count: ordered.length, + })} + +
+ )}
    {rows.map((row, idx) => { const isLast = idx === rows.length - 1; diff --git a/frontend/editor/src/core/components/shared/AppConfigModal.tsx b/frontend/editor/src/core/components/shared/AppConfigModal.tsx index 264db178ac..485403c512 100644 --- a/frontend/editor/src/core/components/shared/AppConfigModal.tsx +++ b/frontend/editor/src/core/components/shared/AppConfigModal.tsx @@ -11,7 +11,11 @@ import { useNavigate, useLocation } from "react-router-dom"; import { useTranslation } from "react-i18next"; import LocalIcon from "@app/components/shared/LocalIcon"; import { useConfigNavSections } from "@app/components/shared/config/configNavSections"; -import { NavKey, VALID_NAV_KEYS } from "@app/components/shared/config/types"; +import { + NavKey, + VALID_NAV_KEYS, + type ConfigNavSection, +} from "@app/components/shared/config/types"; import { useAppConfig } from "@app/contexts/AppConfigContext"; import { COOKIE_CONSENT_SCROLL_SHARD } from "@app/hooks/useCookieConsent"; import "@app/components/shared/AppConfigModal.css"; @@ -31,6 +35,18 @@ import { stripBasePath, withBasePath } from "@app/constants/app"; interface AppConfigModalProps { opened: boolean; onClose: () => void; + /** + * Mirror the active section to /settings/ URLs (deep links, history + * unwind on close). Hosts mounted away from the editor's /settings route β€” + * the admin portal β€” turn this off and the modal keeps its section purely in + * state. + */ + urlSync?: boolean; + /** Section to land on when opening. Only honoured when urlSync is off (URL + * deep links win otherwise). */ + initialSection?: NavKey | null; + /** Host-specific sections appended after the build's registry sections. */ + extraSections?: ConfigNavSection[]; } // Extract section from URL path (e.g., /settings/people -> people) @@ -46,12 +62,18 @@ const getSectionFromPath = (pathname: string): NavKey | null => { const AppConfigModalInner: React.FC = ({ opened, onClose, + urlSync = true, + initialSection, + extraSections, }) => { const { t } = useTranslation(); // Initialize from the URL so a deep link (`/settings/people`) lands on the // right tab without a one-frame "general" flicker. const [active, setActive] = useState( - () => getSectionFromPath(window.location.pathname) ?? "general", + () => + (urlSync ? getSectionFromPath(window.location.pathname) : null) ?? + initialSection ?? + "general", ); const isMobile = useIsMobile(); const navigate = useNavigate(); @@ -66,6 +88,7 @@ const AppConfigModalInner: React.FC = ({ // those update the URL via `history.replaceState` directly and never push // a new React Router location. useEffect(() => { + if (!urlSync) return; const section = getSectionFromPath(location.pathname); if (opened && section) { setActive(section); @@ -77,7 +100,14 @@ const AppConfigModalInner: React.FC = ({ // If at /settings without a section, redirect to general navigate("/settings/general", { replace: true }); } - }, [location.pathname, opened, navigate]); + }, [location.pathname, opened, navigate, urlSync]); + + // Non-URL hosts land the modal on the section they asked for. + useEffect(() => { + if (opened && !urlSync && initialSection) { + setActive(initialSection); + } + }, [opened, urlSync, initialSection]); useEffect(() => { if (opened) { @@ -99,6 +129,7 @@ const AppConfigModalInner: React.FC = ({ const switchSection = useCallback( (key: NavKey) => { setActive(key); + if (!urlSync) return; const alreadyInSettings = stripBasePath( window.location.pathname, ).startsWith("/settings"); @@ -112,7 +143,7 @@ const AppConfigModalInner: React.FC = ({ navigate(`/settings/${key}`); } }, - [navigate], + [navigate, urlSync], ); // Backwards-compat: external `appConfig:navigate` events route through the @@ -156,7 +187,7 @@ const AppConfigModalInner: React.FC = ({ // Only unwind history if settings was opened via the URL; opened via state // there's no /settings entry to pop and navigate(-1) would jump to /files. - if (location.pathname.startsWith("/settings")) { + if (urlSync && location.pathname.startsWith("/settings")) { // "default" key = first entry (deep link/refresh); nothing to pop to. if (location.key === "default") { navigate("/", { replace: true }); @@ -165,7 +196,14 @@ const AppConfigModalInner: React.FC = ({ } } onClose(); - }, [confirmIfDirty, location.key, location.pathname, navigate, onClose]); + }, [ + confirmIfDirty, + location.key, + location.pathname, + navigate, + onClose, + urlSync, + ]); // Synchronous wrapper for contexts (e.g. tour buttons) that need () => void const handleCloseSync = useCallback(() => { @@ -173,12 +211,19 @@ const AppConfigModalInner: React.FC = ({ }, [handleClose]); // Left navigation structure and icons - const configNavSections = useConfigNavSections( + const registrySections = useConfigNavSections( isAdmin, runningEE, loginEnabled, handleCloseSync, ); + const configNavSections = useMemo( + () => + extraSections?.length + ? [...registrySections, ...extraSections] + : registrySections, + [registrySections, extraSections], + ); const activeLabel = useMemo(() => { for (const section of configNavSections) { diff --git a/frontend/editor/src/core/components/shared/AppConfigModalLazy.tsx b/frontend/editor/src/core/components/shared/AppConfigModalLazy.tsx index d242db9a66..1ba57618f0 100644 --- a/frontend/editor/src/core/components/shared/AppConfigModalLazy.tsx +++ b/frontend/editor/src/core/components/shared/AppConfigModalLazy.tsx @@ -1,4 +1,8 @@ import { Suspense, lazy, useEffect, useState } from "react"; +import type { + ConfigNavSection, + NavKey, +} from "@app/components/shared/config/types"; // AppConfigModal pulls in the entire settings UI tree (admin sections, // account, supabase auth flows, etc.). We defer loading until the user first @@ -10,11 +14,20 @@ const AppConfigModal = lazy( interface AppConfigModalLazyProps { opened: boolean; onClose: () => void; + /** See AppConfigModal β€” off for hosts outside the /settings route. */ + urlSync?: boolean; + /** Section to land on when opening (non-URL hosts). */ + initialSection?: NavKey | null; + /** Host-specific sections appended after the build's registry sections. */ + extraSections?: ConfigNavSection[]; } export default function AppConfigModalLazy({ opened, onClose, + urlSync, + initialSection, + extraSections, }: AppConfigModalLazyProps) { const [shouldMount, setShouldMount] = useState(false); @@ -24,7 +37,15 @@ export default function AppConfigModalLazy({ return ( - {shouldMount && } + {shouldMount && ( + + )} ); } diff --git a/frontend/editor/src/core/components/shared/FileDocIcon.tsx b/frontend/editor/src/core/components/shared/FileDocIcon.tsx index 135162c54d..aafaf9aabd 100644 --- a/frontend/editor/src/core/components/shared/FileDocIcon.tsx +++ b/frontend/editor/src/core/components/shared/FileDocIcon.tsx @@ -1,3 +1,5 @@ +import { accentColor } from "@app/utils/accentColors"; + export type FileDocVariant = | "pdf" | "spreadsheet" @@ -7,14 +9,15 @@ export type FileDocVariant = | "code" | "generic"; +// Theme-adaptive accent per file type (shared palette; see utils/accentColors.ts). export const VARIANT_COLORS: Record = { - pdf: "#DC2626", - spreadsheet: "#16a34a", - doc: "#2563eb", - image: "#7c3aed", - archive: "#ea580c", - code: "#0891b2", - generic: "#71717a", + pdf: accentColor("red"), + spreadsheet: accentColor("green"), + doc: accentColor("blue"), + image: accentColor("violet"), + archive: accentColor("orange"), + code: accentColor("cyan"), + generic: accentColor("gray"), }; export function FileDocIcon({ @@ -44,7 +47,7 @@ export function FileDocIcon({ const row1Y = 11, row2Y = 14; interior = ( - + @@ -56,7 +59,7 @@ export function FileDocIcon({ <>
diff --git a/frontend/editor/src/portal/components/procurement/DocumentLedger.tsx b/frontend/editor/src/portal/components/procurement/DocumentLedger.tsx index 8189d006f8..72ab22a341 100644 --- a/frontend/editor/src/portal/components/procurement/DocumentLedger.tsx +++ b/frontend/editor/src/portal/components/procurement/DocumentLedger.tsx @@ -78,7 +78,9 @@ export function DocumentLedger({ {group.label} {blurb && ( - Β· {blurb} + + Β· {t(blurb)} + )} {cur && ( diff --git a/frontend/editor/src/portal/components/procurement/LockedState.stories.tsx b/frontend/editor/src/portal/components/procurement/LockedState.stories.tsx index e88c810801..eae9db4d99 100644 --- a/frontend/editor/src/portal/components/procurement/LockedState.stories.tsx +++ b/frontend/editor/src/portal/components/procurement/LockedState.stories.tsx @@ -1,6 +1,6 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { LockedState } from "@portal/components/procurement/LockedState"; -import { JOURNEY } from "@portal/mocks/procurement"; +import { JOURNEY } from "@portal/api/procurement"; import "@portal/views/Procurement.css"; const meta: Meta = { diff --git a/frontend/editor/src/portal/components/procurement/StageStepper.stories.tsx b/frontend/editor/src/portal/components/procurement/StageStepper.stories.tsx index a27b0b90aa..7adb6951fb 100644 --- a/frontend/editor/src/portal/components/procurement/StageStepper.stories.tsx +++ b/frontend/editor/src/portal/components/procurement/StageStepper.stories.tsx @@ -1,6 +1,6 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { StageStepper } from "@portal/components/procurement/StageStepper"; -import { JOURNEY } from "@portal/mocks/procurement"; +import { JOURNEY } from "@portal/api/procurement"; import "@portal/views/Procurement.css"; const meta: Meta = { diff --git a/frontend/editor/src/portal/components/procurement/StageStepper.tsx b/frontend/editor/src/portal/components/procurement/StageStepper.tsx index 1981078459..3b17f9a0d9 100644 --- a/frontend/editor/src/portal/components/procurement/StageStepper.tsx +++ b/frontend/editor/src/portal/components/procurement/StageStepper.tsx @@ -1,4 +1,5 @@ import { Fragment } from "react"; +import { useTranslation } from "react-i18next"; import type { DealStage, JourneyStep } from "@portal/api/procurement"; /** Status of a step relative to the deal's current stage. */ @@ -19,6 +20,7 @@ export function StageStepper({ currentStage: DealStage; locked?: boolean; }) { + const { t } = useTranslation(); const order = journey.map((s) => s.stage); const curIdx = locked ? -1 : order.indexOf(currentStage); @@ -40,7 +42,7 @@ export function StageStepper({ )}
- {step.label} + {t(step.label)}
); diff --git a/frontend/editor/src/portal/components/settings/accountLinkSettings.tsx b/frontend/editor/src/portal/components/settings/accountLinkSettings.tsx index 8e769672d7..22c03ec682 100644 --- a/frontend/editor/src/portal/components/settings/accountLinkSettings.tsx +++ b/frontend/editor/src/portal/components/settings/accountLinkSettings.tsx @@ -1,26 +1,26 @@ -import type { ComponentType, ReactNode } from "react"; -import { LinkIcon } from "@portal/components/icons"; +import type { ComponentType } from "react"; import { AccountLinkPanel } from "@portal/components/account-link/AccountLinkPanel"; export interface AccountLinkSettingsSeam { - /** Section key in the Settings nav + body switch. */ - navKey: string; + /** Nav key in the shared settings modal (registered in config/types.ts). */ + navKey: "account-link"; /** i18n key for the nav label; resolved with `t()` at the call site. */ labelKey: string; - icon: ReactNode; + /** LocalIcon name for the nav item. */ + icon: string; /** The section body β€” the account-link panel. */ Body: ComponentType; } /** - * The admin "Account link" section of Settings (self-hosted only). The SaaS - * build shadows this file with `null`: the signed-in account IS the SaaS - * account, so there is no instance to link β€” the nav item and its panel both - * drop out, and nothing imports the link-only AccountLinkPanel. + * The admin "Account link" section of the shared settings modal (self-hosted + * only). The SaaS build shadows this file with `null`: the signed-in account IS + * the SaaS account, so there is no instance to link β€” the nav item and its + * panel both drop out, and nothing imports the link-only AccountLinkPanel. */ export const accountLinkSettings: AccountLinkSettingsSeam | null = { navKey: "account-link", labelKey: "portal.settings.sections.account-link", - icon: , + icon: "link-rounded", Body: AccountLinkPanel, }; diff --git a/frontend/editor/src/portal/components/sources/ConnectWizard.test.tsx b/frontend/editor/src/portal/components/sources/ConnectWizard.test.tsx index d354da1d40..2ae0d9fdf3 100644 --- a/frontend/editor/src/portal/components/sources/ConnectWizard.test.tsx +++ b/frontend/editor/src/portal/components/sources/ConnectWizard.test.tsx @@ -61,7 +61,12 @@ describe("ConnectWizard", () => { expect(createSource).toHaveBeenCalledWith({ name: "Claims intake", type: "folder", - options: { directory: "/data/incoming", mode: "consume" }, + options: { + directory: "/data/incoming", + mode: "consume", + recursive: "false", + identity: "stat", + }, enabled: true, }); await waitFor(() => { @@ -101,11 +106,17 @@ describe("ConnectWizard", () => { await waitFor(() => { expect(createSource).toHaveBeenCalledTimes(1); }); + // Options absent from the stored source are submitted at their defaults. expect(createSource).toHaveBeenCalledWith({ id: "s1", name: "James", type: "folder", - options: { directory: "/data/in", mode: "consume" }, + options: { + directory: "/data/in", + mode: "consume", + recursive: "false", + identity: "stat", + }, enabled: true, }); await waitFor(() => { diff --git a/frontend/editor/src/portal/components/sources/sourceTypes.ts b/frontend/editor/src/portal/components/sources/sourceTypes.ts index f3e128ef55..dcae138670 100644 --- a/frontend/editor/src/portal/components/sources/sourceTypes.ts +++ b/frontend/editor/src/portal/components/sources/sourceTypes.ts @@ -91,6 +91,43 @@ export const CREATABLE_SOURCE_TYPES: CreatableSourceType[] = [ }, ], }, + { + key: "recursive", + labelKey: "portal.sources.types.folder.fields.recursive.label", + control: "select", + defaultValue: "false", + options: [ + { + value: "false", + labelKey: + "portal.sources.types.folder.fields.recursive.options.top", + }, + { + value: "true", + labelKey: + "portal.sources.types.folder.fields.recursive.options.all", + }, + ], + }, + { + key: "identity", + labelKey: "portal.sources.types.folder.fields.identity.label", + control: "select", + defaultValue: "stat", + helperTextKey: "portal.sources.types.folder.fields.identity.helperText", + options: [ + { + value: "stat", + labelKey: + "portal.sources.types.folder.fields.identity.options.stat", + }, + { + value: "hash", + labelKey: + "portal.sources.types.folder.fields.identity.options.hash", + }, + ], + }, ], }, ]; diff --git a/frontend/editor/src/portal/components/users/AccessControls.stories.tsx b/frontend/editor/src/portal/components/users/AccessControls.stories.tsx deleted file mode 100644 index 45f69ec8a3..0000000000 --- a/frontend/editor/src/portal/components/users/AccessControls.stories.tsx +++ /dev/null @@ -1,26 +0,0 @@ -import type { Meta, StoryObj } from "@storybook/react-vite"; -import { accessFor } from "@portal/mocks/users"; -import { AccessControls } from "@portal/components/users/AccessControls"; - -const meta: Meta = { - title: "Portal/Users/AccessControls", - component: AccessControls, - parameters: { layout: "padded" }, -}; -export default meta; -type Story = StoryObj; - -/** Free: seat limit + upgrade nudge only. */ -export const Free: Story = { - args: { access: accessFor("free") }, -}; - -/** Pro: adds self-service MFA + session toggles. */ -export const Pro: Story = { - args: { access: accessFor("pro") }, -}; - -/** Enterprise: SSO/SAML, SCIM provisioning and enforced MFA. */ -export const Enterprise: Story = { - args: { access: accessFor("enterprise") }, -}; diff --git a/frontend/editor/src/portal/components/users/AccessControls.tsx b/frontend/editor/src/portal/components/users/AccessControls.tsx deleted file mode 100644 index 9744e13141..0000000000 --- a/frontend/editor/src/portal/components/users/AccessControls.tsx +++ /dev/null @@ -1,199 +0,0 @@ -import { useState } from "react"; -import { useTranslation } from "react-i18next"; -import { - Banner, - Button, - Card, - ProgressBar, - StatTile, - StatusBadge, - ToggleSwitch, -} from "@app/ui"; -import type { AccessControls as Access } from "@portal/api/users"; -import { seatsLabel } from "@portal/components/users/format"; -import "@portal/views/Users.css"; - -interface AccessControlsProps { - access: Access; -} - -/** - * Access posture for the org, scaling by tier: - * free β€” seat limit + upgrade nudge only - * pro β€” adds self-service MFA + session timeout toggles - * enterprise β€” adds SSO/SAML, SCIM provisioning and enforced MFA - * - * Toggles hold local state only; persisting them is a backend wiring task. - */ -export function AccessControls({ access }: AccessControlsProps) { - const { t } = useTranslation(); - const [mfaEnforced, setMfaEnforced] = useState(access.mfaEnforced ?? false); - const [shortSessions, setShortSessions] = useState(false); - - const seatPct = - access.seatLimit === null - ? 0 - : Math.min(1, access.seatsUsed / access.seatLimit); - - return ( -
-
-

- {t("portal.users.access.title")} -

-

- {t("portal.users.access.subtitle")} -

-
- -
- {/* Seats β€” shown on every tier. */} - -
-

- {t("portal.users.access.seats.title")} -

- - {seatsLabel(access.seatsUsed, access.seatLimit)} - -
- {access.seatLimit === null ? ( -

- {t("portal.users.access.seats.unlimited")} -

- ) : ( - - )} -
- - {/* Pro+: MFA + sessions self-service. */} - {access.mfaAvailable && ( - -

- {t("portal.users.access.auth.title")} -

-
-
- { - setMfaEnforced(v); - // TODO(backend): PATCH /v1/users/access { mfaEnforced } - }} - label={t("portal.users.access.auth.requireMfa.label")} - description={ - access.mfaEnforced - ? t("portal.users.access.auth.requireMfa.enforced") - : t("portal.users.access.auth.requireMfa.description") - } - disabled={access.mfaEnforced} - /> -
-
- { - setShortSessions(v); - // TODO(backend): PATCH /v1/users/access { sessionTimeout } - }} - label={t("portal.users.access.auth.shortSessions.label")} - description={t( - "portal.users.access.auth.shortSessions.description", - { - timeout: access.sessionTimeout, - }, - )} - /> -
-
-
- )} - - {/* Enterprise: SSO. */} - {access.sso && ( - -
-

- {t("portal.users.access.sso.title")} -

- - {access.sso.status === "connected" - ? t("portal.users.access.sso.connected") - : t("portal.users.access.sso.notConfigured")} - -
-
- - -
- -
- )} - - {/* Enterprise: SCIM provisioning. */} - {access.scim && ( - -
-

- {t("portal.users.access.scim.title")} -

- - {access.scim.enabled - ? t("portal.users.access.scim.active") - : t("portal.users.access.scim.off")} - -
-
- - -
-

- {t("portal.users.access.scim.note")} -

-
- )} -
- - {/* Free: upgrade nudge spans the section. */} - {access.upgradeHint && ( - - {t("portal.users.access.upgrade.action")} - - } - /> - )} -
- ); -} diff --git a/frontend/editor/src/portal/components/users/ConfirmModal.stories.tsx b/frontend/editor/src/portal/components/users/ConfirmModal.stories.tsx new file mode 100644 index 0000000000..92eeca3f42 --- /dev/null +++ b/frontend/editor/src/portal/components/users/ConfirmModal.stories.tsx @@ -0,0 +1,32 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { ConfirmModal } from "@portal/components/users/ConfirmModal"; + +const meta: Meta = { + title: "Portal/Users/ConfirmModal", + component: ConfirmModal, + parameters: { layout: "fullscreen" }, + args: { + open: true, + title: "Remove member", + body: "Permanently remove Sarah Kowalski from the organization? This cannot be undone.", + confirmLabel: "Remove from org", + danger: true, + onConfirm: () => {}, + onCancel: () => {}, + }, +}; +export default meta; +type Story = StoryObj; + +/** Destructive confirm (red button). */ +export const Danger: Story = {}; + +/** Neutral confirm. */ +export const Neutral: Story = { + args: { + title: "Reset MFA", + body: "Remove this member's MFA enrolment?", + confirmLabel: "Reset MFA", + danger: false, + }, +}; diff --git a/frontend/editor/src/portal/components/users/ConfirmModal.tsx b/frontend/editor/src/portal/components/users/ConfirmModal.tsx new file mode 100644 index 0000000000..183b61bde3 --- /dev/null +++ b/frontend/editor/src/portal/components/users/ConfirmModal.tsx @@ -0,0 +1,54 @@ +import { useTranslation } from "react-i18next"; +import { Button, Modal } from "@app/ui"; +import "@portal/views/Users.css"; + +interface ConfirmModalProps { + open: boolean; + title: string; + body: string; + confirmLabel: string; + /** Red confirm button for destructive actions. */ + danger?: boolean; + busy?: boolean; + onConfirm: () => void; + onCancel: () => void; +} + +/** Small reusable confirm dialog for destructive/irreversible actions. */ +export function ConfirmModal({ + open, + title, + body, + confirmLabel, + danger, + busy, + onConfirm, + onCancel, +}: ConfirmModalProps) { + const { t } = useTranslation(); + return ( + + + + + } + > +

{body}

+
+ ); +} diff --git a/frontend/editor/src/portal/components/users/InviteMemberModal.stories.tsx b/frontend/editor/src/portal/components/users/InviteMemberModal.stories.tsx index cffa89d54d..9d1127d8a5 100644 --- a/frontend/editor/src/portal/components/users/InviteMemberModal.stories.tsx +++ b/frontend/editor/src/portal/components/users/InviteMemberModal.stories.tsx @@ -1,16 +1,72 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { InviteMemberModal } from "@portal/components/users/InviteMemberModal"; +import type { Team } from "@portal/api/teams"; + +const TEAMS: Team[] = [ + { id: 1, name: "Default", userCount: 4, owners: [] }, + { id: 2, name: "Engineering", userCount: 3, owners: ["tom"] }, + { id: 3, name: "Compliance", userCount: 2, owners: ["dana"] }, +]; const meta: Meta = { title: "Portal/Users/InviteMemberModal", component: InviteMemberModal, parameters: { layout: "fullscreen" }, - args: { open: true, onClose: () => {} }, + args: { + open: true, + onClose: () => {}, + onInvited: () => {}, + teams: TEAMS, + // Admin build: the Processor access option is offered (gated on grant management). + manageGrants: true, + }, }; export default meta; type Story = StoryObj; -/** Email + role; Send invite validates locally then closes (demo shell). */ +/** Invite by email with a role, team, and starting access level. */ export const Open: Story = {}; +/** Opened from a team's "Add to team" β€” the team is preselected. */ +export const ScopedToTeam: Story = { + args: { defaultTeamId: 2 }, +}; + +/** Self-hosted with mail configured: the email/create toggle is offered; opens in + * create mode (the default when account creation is available). */ +export const SelfHostedDirectCreate: Story = { + args: { + canDirectCreate: true, + canEmailInvite: true, + hasOauth: true, + hasSaml: true, + }, +}; + +/** Self-hosted without SMTP/invites: no "Invite by email" option at all - just the + * create-account form. */ +export const SelfHostedNoMail: Story = { + args: { + canDirectCreate: true, + canEmailInvite: false, + hasOauth: true, + hasSaml: true, + }, +}; + +/** The direct "Create account" form: username, password, sign-in method, force-MFA. */ +export const CreateAccountForm: Story = { + args: { + canDirectCreate: true, + hasOauth: true, + hasSaml: true, + initialMode: "direct", + }, +}; + +/** SaaS: no "admin" (Org Owner) option and no Processor grant (both admin-only). */ +export const NoAdminRole: Story = { + args: { adminRole: false, manageGrants: false }, +}; + export const Closed: Story = { args: { open: false } }; diff --git a/frontend/editor/src/portal/components/users/InviteMemberModal.test.tsx b/frontend/editor/src/portal/components/users/InviteMemberModal.test.tsx new file mode 100644 index 0000000000..cf43743817 --- /dev/null +++ b/frontend/editor/src/portal/components/users/InviteMemberModal.test.tsx @@ -0,0 +1,72 @@ +import type { ComponentProps } from "react"; +import { describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { MantineProvider } from "@mantine/core"; + +// Deterministic i18n: render the English fallback so assertions read naturally. +vi.mock("react-i18next", () => ({ + useTranslation: () => ({ + t: (key: string, fallback?: string) => fallback ?? key, + i18n: { changeLanguage: vi.fn() }, + }), +})); + +vi.mock("@portal/contexts/TierContext", () => ({ + useTier: () => ({ tier: "pro" }), +})); + +// The modal only calls these on submit; stub so imports resolve and no fetch fires. +vi.mock("@portal/api/users", () => ({ + createMember: vi.fn(), + fetchUsers: vi.fn().mockResolvedValue({ members: [] }), + inviteMember: vi.fn(), + ROLE_LABEL: { member: "Member", admin: "Admin" }, +})); +vi.mock("@portal/api/access", () => ({ createGrant: vi.fn() })); + +import { InviteMemberModal } from "@portal/components/users/InviteMemberModal"; +import type { Team } from "@portal/api/teams"; + +const TEAMS: Team[] = [{ id: 1, name: "Default", userCount: 1, owners: [] }]; + +function renderModal(props: Partial>) { + return render( + + {}} teams={TEAMS} {...props} /> + , + ); +} + +describe("InviteMemberModal β€” add-user method gating", () => { + it("SaaS (email only): no method toggle, opens to the email field", () => { + renderModal({ canDirectCreate: false, canEmailInvite: true }); + expect(screen.queryByText("How to add them")).not.toBeInTheDocument(); + expect(screen.getByText("Email address")).toBeInTheDocument(); + expect(screen.queryByText("Username")).not.toBeInTheDocument(); + }); + + it("self-hosted with mail: offers the toggle and defaults to create-account", () => { + renderModal({ canDirectCreate: true, canEmailInvite: true }); + expect(screen.getByText("How to add them")).toBeInTheDocument(); + // Default mode is create-account (username field, not email). + expect(screen.getByText("Username")).toBeInTheDocument(); + expect(screen.queryByText("Email address")).not.toBeInTheDocument(); + }); + + it("self-hosted without SMTP: no email option at all, create-account only", () => { + renderModal({ canDirectCreate: true, canEmailInvite: false }); + expect(screen.queryByText("How to add them")).not.toBeInTheDocument(); + expect(screen.getByText("Username")).toBeInTheDocument(); + expect(screen.queryByText("Email address")).not.toBeInTheDocument(); + }); + + it("clamps initialMode=email to create-account when email is unavailable", () => { + renderModal({ + canDirectCreate: true, + canEmailInvite: false, + initialMode: "email", + }); + expect(screen.getByText("Username")).toBeInTheDocument(); + expect(screen.queryByText("Email address")).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/editor/src/portal/components/users/InviteMemberModal.tsx b/frontend/editor/src/portal/components/users/InviteMemberModal.tsx index fe90105cf5..515c95923e 100644 --- a/frontend/editor/src/portal/components/users/InviteMemberModal.tsx +++ b/frontend/editor/src/portal/components/users/InviteMemberModal.tsx @@ -1,54 +1,256 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; -import { Button, FormField, Input, Modal, Select } from "@app/ui"; -import { type RoleId, ROLES } from "@portal/api/users"; +import { Button, Checkbox, FormField, Input, Modal, Select } from "@app/ui"; +import { + createMember, + fetchUsers, + inviteMember, + ROLE_LABEL, + type AuthType, +} from "@portal/api/users"; +import { createGrant } from "@portal/api/access"; +import { errorMessage } from "@portal/api/http"; +import type { Team } from "@portal/api/teams"; +import { useTier } from "@portal/contexts/TierContext"; import "@portal/views/Users.css"; interface InviteMemberModalProps { open: boolean; onClose: () => void; + onInvited?: () => void; + teams: Team[]; + defaultTeamId?: number | null; + /** Self-hosted with password login: enables the "Create account" mode. */ + canDirectCreate?: boolean; + /** Whether "Invite by email" is offered. Off when SMTP/invites aren't configured + * (self-hosted); always on for SaaS. */ + canEmailInvite?: boolean; + hasOauth?: boolean; + hasSaml?: boolean; + /** Whether the "admin" (Org Owner) role can be assigned. Off on SaaS. */ + adminRole?: boolean; + /** Whether portal-access grants can be created (ADMIN-only). Gates the Processor option. */ + manageGrants?: boolean; + /** Non-blocking notice back to the parent (e.g. a deferred Processor grant). */ + onNotice?: (message: string) => void; + /** Force the initial mode (mainly for Storybook). Defaults to "direct" when account + * creation is available, else "email". */ + initialMode?: "email" | "direct"; } -const ROLE_SELECT_OPTIONS = ROLES.map((r) => ({ - value: r.id, - label: r.label, -})); +type InviteRole = "member" | "admin"; +type Mode = "email" | "direct"; -/** Org Owner is reserved for transfer flows β€” invites default to Developer. */ -const DEFAULT_ROLE: RoleId = "developer"; +// Values hold i18n keys; resolved with t() where the select renders. +const ROLE_SELECT_OPTIONS: { value: InviteRole; labelKey: string }[] = [ + { value: "member", labelKey: ROLE_LABEL.member }, + { value: "admin", labelKey: ROLE_LABEL.admin }, +]; const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; -/** - * Invite-by-email shell. Submitting validates locally then closes without - * sending β€” wiring the submit to the backend dispatches the invitation. - */ -export function InviteMemberModal({ open, onClose }: InviteMemberModalProps) { +export function InviteMemberModal({ + open, + onClose, + onInvited, + teams, + defaultTeamId, + canDirectCreate = false, + canEmailInvite = true, + hasOauth = false, + hasSaml = false, + adminRole = true, + manageGrants = false, + onNotice, + initialMode, +}: InviteMemberModalProps) { const { t } = useTranslation(); + const { tier } = useTier(); + const [mode, setMode] = useState("email"); const [email, setEmail] = useState(""); - const [role, setRole] = useState(DEFAULT_ROLE); + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + const [authType, setAuthType] = useState("WEB"); + const [forceChange, setForceChange] = useState(true); + const [forceMFA, setForceMFA] = useState(false); + const [role, setRole] = useState("member"); + const [teamId, setTeamId] = useState(""); + const [processor, setProcessor] = useState(false); const [touched, setTouched] = useState(false); + const [sending, setSending] = useState(false); + const [submitError, setSubmitError] = useState(null); + + // Which add-user modes this flavor/config offers. Self-hosted offers direct create; + // SaaS offers email; self-hosted also offers email once SMTP + invites are configured. + const directAvailable = canDirectCreate; + const emailAvailable = canEmailInvite; + const preferredMode: Mode = directAvailable ? "direct" : "email"; + + useEffect(() => { + if (!open) return; + // Honor an explicit initialMode only when that mode is actually available. + const requested: Mode = + initialMode && + (initialMode === "email" ? emailAvailable : directAvailable) + ? initialMode + : preferredMode; + setMode(requested); + setTeamId( + defaultTeamId != null + ? String(defaultTeamId) + : teams.length > 0 + ? String(teams[0].id) + : "", + ); + }, [ + open, + defaultTeamId, + teams, + directAvailable, + emailAvailable, + preferredMode, + initialMode, + ]); + + // A server-side submit error must not outlive the input that caused it. + useEffect(() => { + setSubmitError(null); + }, [email, username, password]); + + // Drop the "admin" (Org Owner) option where it can't be assigned (SaaS). + const roleOptions = ( + adminRole + ? ROLE_SELECT_OPTIONS + : ROLE_SELECT_OPTIONS.filter((o) => o.value !== "admin") + ).map((o) => ({ value: o.value, label: t(o.labelKey) })); + + const authTypeOptions: { value: AuthType; label: string }[] = [ + { value: "WEB", label: t("users.invite.authWeb", "Password") }, + ...(hasOauth + ? [ + { + value: "OAUTH2" as AuthType, + label: t("users.invite.authOauth", "OAuth2 / SSO"), + }, + ] + : []), + ...(hasSaml + ? [ + { + value: "SAML2" as AuthType, + label: t("users.invite.authSaml", "SAML 2.0"), + }, + ] + : []), + ]; const emailValid = EMAIL_RE.test(email.trim()); + const usernameValid = username.trim().length >= 3; + const needsPassword = mode === "direct" && authType === "WEB"; + const passwordValid = !needsPassword || password.length >= 8; + const error = - touched && !emailValid ? t("portal.users.invite.emailError") : undefined; + (touched && mode === "email" && !emailValid + ? t("users.invite.emailError", "Enter a valid email address") + : undefined) ?? + (touched && mode === "direct" && !usernameValid + ? t( + "users.invite.usernameError", + "Username must be at least 3 characters", + ) + : undefined) ?? + (touched && mode === "direct" && !passwordValid + ? t( + "users.invite.passwordError", + "Password must be at least 8 characters", + ) + : undefined) ?? + submitError ?? + undefined; function close() { onClose(); - // Reset for the next open, after the close transition has finished. setTimeout(() => { setEmail(""); - setRole(DEFAULT_ROLE); + setUsername(""); + setPassword(""); + setAuthType("WEB"); + setForceChange(true); + setForceMFA(false); + setRole("member"); + setProcessor(false); setTouched(false); + setSubmitError(null); }, 200); } - function submit() { + // Grant Processor to the just-added user. Returns false when it couldn't be applied + // (an email invitee usually isn't a resolvable user yet, or the grant call is denied). + async function grantProcessor( + match: (m: { email: string; username?: string }) => boolean, + ): Promise { + const { members } = await fetchUsers(tier); + const user = members.find(match); + if (!user) return false; + await createGrant({ + resourceType: "PORTAL", + resourceId: "", + principalType: "USER", + principalId: Number(user.id), + permission: "USE", + }); + return true; + } + + async function submit() { setTouched(true); - if (!emailValid) return; - // TODO(backend): POST /v1/users/invitations { email, role } β€” send the - // invite, then close on success and refetch the members list. - close(); + setSubmitError(null); + if (sending) return; + const teamNum = teamId ? Number(teamId) : undefined; + setSending(true); + try { + let processorApplied = true; + if (mode === "email") { + if (!emailValid) return; + const result = await inviteMember(email.trim(), role, teamNum); + if (result?.error || result?.errors) { + setSubmitError(result.error ?? result.errors ?? null); + return; + } + if (processor) + processorApplied = await grantProcessor( + (m) => m.email === email.trim() || m.username === email.trim(), + ).catch(() => false); + } else { + if (!usernameValid || !passwordValid) return; + const created = await createMember({ + username: username.trim(), + password: authType === "WEB" ? password : undefined, + role, + teamId: teamNum, + authType, + forceChange, + forceMFA, + }); + if (processor) + processorApplied = await grantProcessor( + (m) => m.username === created, + ).catch(() => false); + } + if (processor && !processorApplied) + onNotice?.( + t( + "users.invite.processorDeferred", + "Invite sent, but Processor access couldn't be granted yet - set it from the roster once they've joined.", + ), + ); + onInvited?.(); + close(); + } catch (e) { + setSubmitError(errorMessage(e)); + } finally { + setSending(false); + } } return ( @@ -56,44 +258,184 @@ export function InviteMemberModal({ open, onClose }: InviteMemberModalProps) { open={open} onClose={close} width="sm" - title={t("portal.common.inviteMember")} - subtitle={t("portal.users.invite.subtitle")} + title={ + mode === "direct" + ? t("users.invite.createTitle", "Create account") + : t("users.invite.title", "Invite people") + } + subtitle={ + mode === "direct" + ? t( + "users.invite.createSubtitle", + "Create a self-hosted account with a password or SSO.", + ) + : t( + "users.invite.subtitle2", + "They'll get an email to join your Stirling workspace.", + ) + } footer={
-
} >
- - setEmail(e.target.value)} - onBlur={() => setTouched(true)} + {directAvailable && emailAvailable && ( + + setEmail(e.target.value)} + onBlur={() => setTouched(true)} + /> + + ) : ( + <> + + setUsername(e.target.value)} + onBlur={() => setTouched(true)} + /> + + {authTypeOptions.length > 1 && ( + + setPassword(e.target.value)} + /> + + )} + + )} + +
+ + ({ + value: String(tm.id), + label: tm.name, + }))} + value={teamId} + onChange={(value) => setTeamId(value ?? "")} + /> + +
+ + {mode === "direct" && ( +
+ setForceChange(e.target.checked)} + label={t( + "users.invite.forceChange", + "Require a password change on first login", + )} + disabled={authType !== "WEB"} + /> + setForceMFA(e.target.checked)} + label={t( + "users.invite.forceMfa", + "Require MFA setup on first login", + )} + /> +
+ )} + +
+ + {t("users.invite.access", "Access")} + + - - - setTeamId(value ?? "")} + /> + + {error && ( +

+ {error} +

+ )} +
+ + ); +} diff --git a/frontend/editor/src/portal/components/users/NewTeamModal.stories.tsx b/frontend/editor/src/portal/components/users/NewTeamModal.stories.tsx new file mode 100644 index 0000000000..ce0b5409ce --- /dev/null +++ b/frontend/editor/src/portal/components/users/NewTeamModal.stories.tsx @@ -0,0 +1,18 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { NewTeamModal } from "@portal/components/users/NewTeamModal"; + +const meta: Meta = { + title: "Portal/Users/NewTeamModal", + component: NewTeamModal, + parameters: { layout: "fullscreen" }, + args: { + open: true, + onClose: () => {}, + onCreated: () => {}, + }, +}; +export default meta; +type Story = StoryObj; + +/** Create a team and (optionally) invite its owner. */ +export const Default: Story = {}; diff --git a/frontend/editor/src/portal/components/users/NewTeamModal.tsx b/frontend/editor/src/portal/components/users/NewTeamModal.tsx new file mode 100644 index 0000000000..cc7cf26e81 --- /dev/null +++ b/frontend/editor/src/portal/components/users/NewTeamModal.tsx @@ -0,0 +1,158 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button, FormField, Input, Modal } from "@app/ui"; +import { errorMessage } from "@portal/api/http"; +import { createTeam, fetchTeams, setTeamOwner } from "@portal/api/teams"; +import { fetchUsers, inviteMember } from "@portal/api/users"; +import { useTier } from "@portal/contexts/TierContext"; +import "@portal/views/Users.css"; + +interface NewTeamModalProps { + open: boolean; + onClose: () => void; + /** Called after a team is created so the roster refetches. */ + onCreated: () => void; +} + +const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +/** + * Create a team and optionally invite its owner. The team is created first, so + * even if the owner invite needs mail (unavailable) the team still lands and an + * owner can be assigned from the roster. + */ +export function NewTeamModal({ open, onClose, onCreated }: NewTeamModalProps) { + const { t } = useTranslation(); + const { tier } = useTier(); + const [name, setName] = useState(""); + const [ownerEmail, setOwnerEmail] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + const [note, setNote] = useState(null); + + function close() { + onClose(); + setTimeout(() => { + setName(""); + setOwnerEmail(""); + setError(null); + setNote(null); + }, 200); + } + + async function inviteOwner(teamName: string, email: string) { + const teams = await fetchTeams(); + const team = teams.find((tm) => tm.name === teamName); + if (!team) return; + await inviteMember(email, "member", team.id); + const { members } = await fetchUsers(tier); + const owner = members.find( + (m) => m.email === email || m.username === email, + ); + if (owner) await setTeamOwner(team.id, owner.id); + } + + async function submit() { + setError(null); + setNote(null); + if (!name.trim()) { + setError(t("users.newTeam.nameRequired", "Team name is required")); + return; + } + const email = ownerEmail.trim(); + if (email && !EMAIL_RE.test(email)) { + setError(t("users.newTeam.emailError", "Enter a valid email address")); + return; + } + setSaving(true); + try { + await createTeam(name.trim()); + let ownerFailed = false; + if (email) { + try { + await inviteOwner(name.trim(), email); + } catch { + // Team is created; keep the modal open so this recovery note stays + // readable (owner invite needs mail, can be redone from the roster). + ownerFailed = true; + setNote( + t( + "users.newTeam.ownerFailed", + "Team created, but the owner couldn't be invited. Assign one from the roster.", + ), + ); + } + } + onCreated(); + if (!ownerFailed) close(); + } catch (e) { + setError(errorMessage(e)); + } finally { + setSaving(false); + } + } + + return ( + + + +
+ } + > +
+ + setName(e.target.value)} + /> + + + setOwnerEmail(e.target.value)} + /> + + {note && ( +

+ {note} +

+ )} + {error && ( +

+ {error} +

+ )} +
+ + ); +} diff --git a/frontend/editor/src/portal/components/users/RenameTeamModal.stories.tsx b/frontend/editor/src/portal/components/users/RenameTeamModal.stories.tsx new file mode 100644 index 0000000000..6e8cba5aea --- /dev/null +++ b/frontend/editor/src/portal/components/users/RenameTeamModal.stories.tsx @@ -0,0 +1,19 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { RenameTeamModal } from "@portal/components/users/RenameTeamModal"; + +const meta: Meta = { + title: "Portal/Users/RenameTeamModal", + component: RenameTeamModal, + parameters: { layout: "fullscreen" }, + args: { + open: true, + teamId: 2, + currentName: "Engineering", + onClose: () => {}, + onDone: () => {}, + }, +}; +export default meta; +type Story = StoryObj; + +export const Default: Story = {}; diff --git a/frontend/editor/src/portal/components/users/RenameTeamModal.tsx b/frontend/editor/src/portal/components/users/RenameTeamModal.tsx new file mode 100644 index 0000000000..d9f1f4b66c --- /dev/null +++ b/frontend/editor/src/portal/components/users/RenameTeamModal.tsx @@ -0,0 +1,83 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button, FormField, Input, Modal } from "@app/ui"; +import { renameTeam } from "@portal/api/teams"; +import { errorMessage } from "@portal/api/http"; +import "@portal/views/Users.css"; + +interface RenameTeamModalProps { + open: boolean; + teamId: number | null; + currentName: string; + onClose: () => void; + onDone: () => void; +} + +/** Rename a team. */ +export function RenameTeamModal({ + open, + teamId, + currentName, + onClose, + onDone, +}: RenameTeamModalProps) { + const { t } = useTranslation(); + const [name, setName] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!open) return; + setName(currentName); + setError(null); + }, [open, currentName]); + + async function submit() { + if (teamId == null || !name.trim()) return; + setSaving(true); + setError(null); + try { + await renameTeam(teamId, name.trim()); + onDone(); + onClose(); + } catch (e) { + setError(errorMessage(e)); + } finally { + setSaving(false); + } + } + + return ( + + + + + } + > +
+ + setName(e.target.value)} /> + + {error && ( +

+ {error} +

+ )} +
+
+ ); +} diff --git a/frontend/editor/src/portal/components/users/ResetPasswordModal.stories.tsx b/frontend/editor/src/portal/components/users/ResetPasswordModal.stories.tsx new file mode 100644 index 0000000000..24a6f58308 --- /dev/null +++ b/frontend/editor/src/portal/components/users/ResetPasswordModal.stories.tsx @@ -0,0 +1,34 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { ResetPasswordModal } from "@portal/components/users/ResetPasswordModal"; +import type { Member } from "@portal/api/users"; + +const MEMBER: Member = { + id: "3", + name: "Sarah Kowalski", + email: "sarah@stirlingpdf.com", + role: "member", + status: "active", + lastActive: "30m ago", + username: "sarah", +}; + +const meta: Meta = { + title: "Portal/Users/ResetPasswordModal", + component: ResetPasswordModal, + parameters: { layout: "fullscreen" }, + args: { + open: true, + member: MEMBER, + mailEnabled: false, + onClose: () => {}, + onDone: () => {}, + }, +}; +export default meta; +type Story = StoryObj; + +/** Auto-generate a secure password (copy + regenerate). */ +export const Default: Story = {}; + +/** With SMTP configured, the admin can email the reset. */ +export const WithEmail: Story = { args: { mailEnabled: true } }; diff --git a/frontend/editor/src/portal/components/users/ResetPasswordModal.tsx b/frontend/editor/src/portal/components/users/ResetPasswordModal.tsx new file mode 100644 index 0000000000..d1cfe9e739 --- /dev/null +++ b/frontend/editor/src/portal/components/users/ResetPasswordModal.tsx @@ -0,0 +1,211 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button, Checkbox, FormField, Input, Modal } from "@app/ui"; +import { resetMemberPassword, type Member } from "@portal/api/users"; +import { errorMessage } from "@portal/api/http"; +import "@portal/views/Users.css"; + +interface ResetPasswordModalProps { + open: boolean; + member: Member | null; + /** SMTP configured - gates the "email the password" options. */ + mailEnabled: boolean; + onClose: () => void; + onDone: () => void; +} + +const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +/** Cryptographically secure password with rejection sampling (no modulo bias). */ +function generatePassword(len = 16): string { + const chars = + "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz23456789!@#$%^&*"; + const max = Math.floor(256 / chars.length) * chars.length; + const out: string[] = []; + const buf = new Uint8Array(1); + while (out.length < len) { + crypto.getRandomValues(buf); + if (buf[0] < max) out.push(chars[buf[0] % chars.length]); + } + return out.join(""); +} + +/** Admin reset of a member's password: auto-generate (with copy) or set manually. */ +export function ResetPasswordModal({ + open, + member, + mailEnabled, + onClose, + onDone, +}: ResetPasswordModalProps) { + const { t } = useTranslation(); + const [autoGenerate, setAutoGenerate] = useState(true); + const [generated, setGenerated] = useState(""); + const [password, setPassword] = useState(""); + const [confirm, setConfirm] = useState(""); + const [forceChange, setForceChange] = useState(true); + const [sendEmail, setSendEmail] = useState(false); + const [includePassword, setIncludePassword] = useState(false); + const [copied, setCopied] = useState(false); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + const canEmail = mailEnabled && EMAIL_RE.test(member?.email ?? ""); + + useEffect(() => { + if (!open) return; + setAutoGenerate(true); + setGenerated(generatePassword()); + setPassword(""); + setConfirm(""); + setForceChange(true); + setSendEmail(false); + setIncludePassword(false); + setCopied(false); + setError(null); + }, [open]); + + function copy() { + void navigator.clipboard?.writeText(generated)?.then(() => setCopied(true)); + } + + async function submit() { + if (!member) return; + setError(null); + const newPassword = autoGenerate ? generated : password; + if (!autoGenerate) { + if (newPassword.length < 8) { + setError( + t("users.resetPw.tooShort", "Password must be at least 8 characters"), + ); + return; + } + if (newPassword !== confirm) { + setError(t("users.resetPw.mismatch", "Passwords do not match")); + return; + } + } + setSaving(true); + try { + await resetMemberPassword(member, { + newPassword, + forcePasswordChange: forceChange, + sendEmail: canEmail && sendEmail, + includePassword: canEmail && sendEmail && includePassword, + }); + onDone(); + onClose(); + } catch (e) { + setError(errorMessage(e)); + } finally { + setSaving(false); + } + } + + return ( + + + + + } + > +
+ setAutoGenerate(e.target.checked)} + label={t("users.resetPw.generate", "Generate a secure password")} + /> + + {autoGenerate ? ( + +
+ + + +
+
+ ) : ( + <> + + setPassword(e.target.value)} + /> + + + setConfirm(e.target.value)} + /> + + + )} + + setForceChange(e.target.checked)} + label={t( + "users.resetPw.forceChange", + "Require a password change on next login", + )} + /> + + {canEmail && ( + <> + setSendEmail(e.target.checked)} + label={t("users.resetPw.email", "Email the user about the reset")} + /> + setIncludePassword(e.target.checked)} + label={t( + "users.resetPw.includePw", + "Include the new password in the email", + )} + /> + + )} + + {error && ( +

+ {error} +

+ )} +
+
+ ); +} diff --git a/frontend/editor/src/portal/components/users/RolesGrid.stories.tsx b/frontend/editor/src/portal/components/users/RolesGrid.stories.tsx deleted file mode 100644 index 386e61fc99..0000000000 --- a/frontend/editor/src/portal/components/users/RolesGrid.stories.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import type { Meta, StoryObj } from "@storybook/react-vite"; -import { ROLES } from "@portal/mocks/users"; -import { RolesGrid } from "@portal/components/users/RolesGrid"; - -const meta: Meta = { - title: "Portal/Users/RolesGrid", - component: RolesGrid, - parameters: { layout: "padded" }, - args: { roles: ROLES }, -}; -export default meta; -type Story = StoryObj; - -/** The five org roles, most β†’ least privileged. */ -export const Default: Story = {}; diff --git a/frontend/editor/src/portal/components/users/RolesGrid.tsx b/frontend/editor/src/portal/components/users/RolesGrid.tsx deleted file mode 100644 index 24527f35be..0000000000 --- a/frontend/editor/src/portal/components/users/RolesGrid.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { useTranslation } from "react-i18next"; -import { Card, Chip } from "@app/ui"; -import type { Role } from "@portal/api/users"; -import { chipAccentForRole } from "@portal/components/users/format"; -import "@portal/views/Users.css"; - -interface RolesGridProps { - roles: Role[]; -} - -/** Reference catalogue of the org roles and what each one can do. */ -export function RolesGrid({ roles }: RolesGridProps) { - const { t } = useTranslation(); - return ( -
-
-

- {t("portal.users.roles.title")} -

-

- {t("portal.users.roles.subtitle")} -

-
-
- {roles.map((role) => ( - -
- - {role.label} - -
-

{role.summary}

-
    - {role.permissions.map((perm) => ( -
  • {perm}
  • - ))} -
-
- ))} -
-
- ); -} diff --git a/frontend/editor/src/portal/components/users/UsersDirectory.stories.tsx b/frontend/editor/src/portal/components/users/UsersDirectory.stories.tsx new file mode 100644 index 0000000000..b7fb277854 --- /dev/null +++ b/frontend/editor/src/portal/components/users/UsersDirectory.stories.tsx @@ -0,0 +1,324 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { UsersDirectory } from "@portal/components/users/UsersDirectory"; +import type { Member } from "@portal/api/users"; +import type { Team } from "@portal/api/teams"; +import type { UsersCapabilities } from "@portal/api/usersCapabilities"; + +/** Self-hosted org-admin: the full action set. */ +const FULL_CAPS: UsersCapabilities = { + orgGroup: true, + changeRole: true, + adminRole: true, + createTeam: true, + deleteTeam: true, + renameTeam: true, + emailInvite: true, + directCreate: true, + resetPassword: true, + unlock: true, + resetMfa: true, + suspend: true, + moveTeam: true, + seats: false, + manageGrants: true, + removeScope: "org", +}; + +/** SaaS team-leader: invite / rename / remove-member only, no org group. */ +const SAAS_CAPS: UsersCapabilities = { + orgGroup: false, + changeRole: false, + adminRole: false, + createTeam: false, + deleteTeam: false, + renameTeam: true, + emailInvite: true, + directCreate: false, + resetPassword: false, + unlock: false, + resetMfa: false, + suspend: false, + moveTeam: false, + seats: true, + manageGrants: false, + removeScope: "team", +}; + +/** A full org: one org owner and two teams, each with a leader. */ +const MEMBERS: Member[] = [ + { + id: "1", + name: "Matt Joseph", + email: "matt@stirlingpdf.com", + role: "admin", + status: "active", + lastActive: "Now", + username: "matt", + teamId: 1, + teamName: "Default", + isSelf: true, + portalAccess: "admin", + }, + { + id: "2", + name: "Tom Reilly", + email: "tom@stirlingpdf.com", + role: "team_owner", + status: "active", + lastActive: "12m ago", + username: "tom", + teamId: 2, + teamName: "Engineering", + teamLead: true, + portalAccess: "role", + }, + { + id: "3", + name: "Sarah Kowalski", + email: "sarah@stirlingpdf.com", + role: "member", + status: "active", + lastActive: "30m ago", + username: "sarah", + teamId: 2, + teamName: "Engineering", + portalAccess: "granted", + portalGrantId: 10, + }, + { + id: "4", + name: "Priya Patel", + email: "priya@stirlingpdf.com", + role: "member", + status: "active", + lastActive: "Never", + username: "priya", + teamId: 2, + teamName: "Engineering", + portalAccess: "none", + }, + { + id: "5", + name: "Dana Okafor", + email: "dana@stirlingpdf.com", + role: "team_owner", + status: "active", + lastActive: "1h ago", + username: "dana", + teamId: 3, + teamName: "Compliance", + teamLead: true, + portalAccess: "role", + }, + { + id: "6", + name: "Lars Eriksson", + email: "lars@stirlingpdf.com", + role: "member", + status: "active", + lastActive: "3h ago", + username: "lars", + teamId: 3, + teamName: "Compliance", + portalAccess: "granted", + portalGrantId: 11, + }, +]; + +/** An external guest (parked concept, shown via showGuests). */ +const GUEST: Member = { + id: "7", + name: "Meridian Legal", + email: "legal@meridian-partners.com", + role: "guest", + status: "active", + lastActive: "2d ago", + username: "legal", + teamId: 1, + teamName: "Default", + portalAccess: "none", +}; + +/** Members exercising the status states: suspended, locked, and MFA-enrolled. */ +const STATE_MEMBERS: Member[] = [ + { + id: "1", + name: "Matt Joseph", + email: "matt@stirlingpdf.com", + role: "admin", + status: "active", + lastActive: "Now", + username: "matt", + teamId: 1, + teamName: "Default", + isSelf: true, + portalAccess: "admin", + }, + { + id: "s1", + name: "Nadia Costa", + email: "nadia@acme.com", + role: "member", + status: "suspended", + lastActive: "12 days ago", + username: "nadia", + teamId: 2, + teamName: "Engineering", + portalAccess: "none", + }, + { + id: "s2", + name: "Leo Fischer", + email: "leo@acme.com", + role: "member", + status: "active", + lastActive: "1h ago", + username: "leo", + teamId: 2, + teamName: "Engineering", + portalAccess: "granted", + portalGrantId: 20, + locked: true, + }, + { + id: "s3", + name: "Aisha Rahman", + email: "aisha@acme.com", + role: "team_owner", + status: "active", + lastActive: "3m ago", + username: "aisha", + teamId: 2, + teamName: "Engineering", + teamLead: true, + portalAccess: "role", + mfaEnabled: true, + }, +]; + +/** A big team (>8) to exercise the "Show all" / "Show less" expander. */ +const BIG_MEMBERS: Member[] = [ + { + id: "b0", + name: "Matt Joseph", + email: "matt@acme.com", + role: "admin", + status: "active", + lastActive: "Now", + username: "matt", + teamId: 1, + isSelf: true, + portalAccess: "admin", + }, + ...Array.from( + { length: 11 }, + (_, i): Member => ({ + id: `big-${i}`, + name: `Teammate ${i + 1}`, + email: `teammate${i + 1}@acme.com`, + role: i === 0 ? "team_owner" : "member", + status: "active", + lastActive: `${i + 1}h ago`, + username: `tm${i + 1}`, + teamId: 9, + teamName: "Platform", + teamLead: i === 0, + portalAccess: i === 0 ? "role" : "none", + }), + ), +]; + +const TEAMS: Team[] = [ + { id: 2, name: "Engineering", userCount: 3, owners: ["tom"] }, + { id: 3, name: "Compliance", userCount: 2, owners: ["dana"] }, +]; + +const meta: Meta = { + title: "Portal/Users/UsersDirectory", + component: UsersDirectory, + parameters: { layout: "padded" }, + args: { + members: MEMBERS, + teams: TEAMS, + capabilities: FULL_CAPS, + onChangeRole: () => {}, + onGrantProcessor: () => {}, + onRevokeProcessor: () => {}, + processorTeamIds: new Set(), + onGrantTeamProcessor: () => {}, + onRevokeTeamProcessor: () => {}, + onAddToTeam: () => {}, + onResetPassword: () => {}, + onMoveToTeam: () => {}, + onToggleEnabled: () => {}, + onUnlock: () => {}, + onDisableMfa: () => {}, + onRemove: () => {}, + onRenameTeam: () => {}, + onDeleteTeam: () => {}, + // Documents the intended "Approves policy" chip; hidden in the live app. + showApprover: true, + }, +}; +export default meta; +type Story = StoryObj; + +/** Organization owner plus two teams, each with a leader. */ +export const Default: Story = {}; + +/** + * SaaS build (team-leader scope): no Organization group, no role select, no + * password/suspend actions - just invite / rename / remove-from-team. + */ +export const SaasTeamLeader: Story = { + args: { + members: MEMBERS.filter((m) => m.role !== "admin"), + capabilities: SAAS_CAPS, + }, +}; + +/** A solo workspace: just the org owner, no teams yet. */ +export const OrgOnly: Story = { + args: { members: MEMBERS.filter((m) => m.role === "admin"), teams: [] }, +}; + +/** A team with a team-wide Processor grant: every member inherits it (solid, non-removable). */ +export const TeamWideProcessor: Story = { + args: { + members: MEMBERS.map((m) => + m.teamId === 2 && m.role === "member" + ? { ...m, portalAccess: "team" as const, portalGrantId: undefined } + : m, + ), + processorTeamIds: new Set([2]), + }, +}; + +/** One team, mixed Processor access (implicit, granted, and not granted). */ +export const SingleTeam: Story = { + args: { + members: MEMBERS.filter((m) => m.teamId === 2 || m.role === "admin"), + teams: TEAMS.filter((t) => t.id === 2), + }, +}; + +/** Guests group + "Guest" role option (parked in the live app; behind showGuests). */ +export const WithGuests: Story = { + args: { members: [...MEMBERS, GUEST], showGuests: true }, +}; + +/** Suspended / locked / MFA-enrolled members show inline tags and gated kebab actions. */ +export const MemberStates: Story = { + args: { + members: STATE_MEMBERS, + teams: TEAMS.filter((t) => t.id === 2), + }, +}; + +/** A team past the collapse limit surfaces the "Show all" expander. */ +export const LargeTeam: Story = { + args: { + members: BIG_MEMBERS, + teams: [{ id: 9, name: "Platform", userCount: 11, owners: ["tm1"] }], + }, +}; diff --git a/frontend/editor/src/portal/components/users/UsersDirectory.tsx b/frontend/editor/src/portal/components/users/UsersDirectory.tsx new file mode 100644 index 0000000000..1419026453 --- /dev/null +++ b/frontend/editor/src/portal/components/users/UsersDirectory.tsx @@ -0,0 +1,478 @@ +import { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Menu } from "@mantine/core"; +import PersonAddAltRounded from "@mui/icons-material/PersonAddAltRounded"; +import { Avatar, Chip, Select } from "@app/ui"; +import { type Member, type RoleId } from "@portal/api/users"; +import type { Team } from "@portal/api/teams"; +import type { UsersCapabilities } from "@portal/api/usersCapabilities"; +import { avatarToneForMember } from "@portal/components/users/format"; +import { + buildDirectory, + type TeamGroup, +} from "@portal/components/users/directory"; +import "@portal/views/Users.css"; + +/** Collapse a group's rows past this many, behind a "Show all" expander. */ +const COLLAPSED_LIMIT = 8; + +/** Teams that can't be renamed/deleted (system-managed). */ +const SYSTEM_TEAMS = new Set(["Default", "Internal"]); + +interface UsersDirectoryProps { + members: Member[]; + teams: Team[]; + /** Flavor-specific action set (self-hosted org-admin vs SaaS team-leader). */ + capabilities: UsersCapabilities; + onChangeRole: (member: Member, role: RoleId) => void; + onGrantProcessor: (member: Member) => void; + onRevokeProcessor: (member: Member) => void; + /** Team ids holding a team-wide Processor grant; members inherit it. */ + processorTeamIds: Set; + onGrantTeamProcessor: (team: TeamGroup) => void; + onRevokeTeamProcessor: (team: TeamGroup) => void; + onAddToTeam: (team: TeamGroup) => void; + // Per-member admin actions (the row kebab). + onResetPassword: (member: Member) => void; + onMoveToTeam: (member: Member) => void; + onToggleEnabled: (member: Member) => void; + onUnlock: (member: Member) => void; + onDisableMfa: (member: Member) => void; + onRemove: (member: Member) => void; + // Team actions (the team-header kebab). + onRenameTeam: (team: TeamGroup) => void; + onDeleteTeam: (team: TeamGroup) => void; + /** + * Show the "Approves policy" capability chip on org owners. Off in the live + * app (no backend for it yet); on in Storybook to document the intended design. + */ + showApprover?: boolean; + /** + * Show the Guests group + the "Guest" role option. Off in the live app (the + * Guest concept is parked); on in Storybook to document the intended design. + */ + showGuests?: boolean; +} + +/** + * The people roster, grouped like the org chart: Organization owners, then each + * team (with its leader), each row carrying capability chips, a role selector, + * and a kebab of admin actions. Long groups collapse behind a "Show all" so big + * orgs stay scannable. + */ +export function UsersDirectory({ + members, + teams, + capabilities, + onChangeRole, + onGrantProcessor, + onRevokeProcessor, + processorTeamIds, + onGrantTeamProcessor, + onRevokeTeamProcessor, + onAddToTeam, + onResetPassword, + onMoveToTeam, + onToggleEnabled, + onUnlock, + onDisableMfa, + onRemove, + onRenameTeam, + onDeleteTeam, + showApprover = false, + showGuests = false, +}: UsersDirectoryProps) { + const { t } = useTranslation(); + const dir = useMemo(() => buildDirectory(members, teams), [members, teams]); + const [expanded, setExpanded] = useState>(new Set()); + + const nameByUsername = useMemo(() => { + const m = new Map(); + for (const member of members) { + if (member.username) m.set(member.username, member.name); + } + return m; + }, [members]); + + const roleOptions: { value: RoleId; label: string }[] = [ + // No SaaS user is ever ROLE_ADMIN, so the Org Owner option is dropped there. + ...(capabilities.adminRole + ? [ + { + value: "admin" as RoleId, + label: t("users.role.orgOwner", "Org Owner"), + }, + ] + : []), + { value: "team_owner", label: t("users.role.teamOwner", "Team Owner") }, + { value: "member", label: t("users.role.member", "Member") }, + ...(showGuests + ? [{ value: "guest" as RoleId, label: t("users.role.guest", "Guest") }] + : []), + ]; + + function toggleExpand(key: string) { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(key)) next.delete(key); + else next.add(key); + return next; + }); + } + + function ownerNames(owners: string[]): string { + return owners.map((u) => nameByUsername.get(u) ?? u).join(", "); + } + + // Whether the team-header kebab has any actions (else it isn't rendered). + function teamKebabHasItems(team: TeamGroup): boolean { + return ( + capabilities.manageGrants || + (!SYSTEM_TEAMS.has(team.name) && + (capabilities.renameTeam || capabilities.deleteTeam)) + ); + } + + // Whether any action sits above the "Remove" item (so we render a divider). + function rowKebabHasUpperActions(m: Member): boolean { + return ( + capabilities.resetPassword || + capabilities.moveTeam || + capabilities.suspend || + (capabilities.unlock && !!m.locked) || + (capabilities.resetMfa && !!m.mfaEnabled) + ); + } + + function rowKebab(m: Member) { + // Removal is org-delete (admin-only) - the only backed path. SaaS "remove from + // team" has no endpoint, so don't offer a control that would 403 or org-delete. + const canRemove = capabilities.removeScope === "org"; + if (!rowKebabHasUpperActions(m) && !canRemove) return null; + return ( + + + + + + {capabilities.resetPassword && ( + onResetPassword(m)}> + {t("users.action.resetPw", "Reset password")} + + )} + {capabilities.moveTeam && ( + onMoveToTeam(m)}> + {t("users.action.move", "Move to team")} + + )} + {capabilities.suspend && ( + onToggleEnabled(m)}> + {m.status === "suspended" + ? t("users.action.reinstate", "Reinstate") + : t("users.action.suspend", "Suspend")} + + )} + {capabilities.unlock && m.locked && ( + onUnlock(m)}> + {t("users.action.unlock", "Unlock account")} + + )} + {capabilities.resetMfa && m.mfaEnabled && ( + onDisableMfa(m)}> + {t("users.action.disableMfa", "Reset MFA")} + + )} + {canRemove && rowKebabHasUpperActions(m) && } + {canRemove && ( + onRemove(m)} + > + {t("users.action.remove", "Remove from org")} + + )} + + + ); + } + + function renderRow(m: Member) { + const access = m.portalAccess ?? "none"; + return ( +
+
+ +
+ + {m.name} + {m.isSelf && ( + + {" "} + {t("users.you", "(you)")} + + )} + {m.status === "suspended" && ( + + {t("users.suspended", "Suspended")} + + )} + {m.locked && ( + + {t("users.locked", "Locked")} + + )} + + {m.email !== m.name && ( + {m.email} + )} +
+
+ +
+ + {t("users.cap.editor", "Editor")} + + {access === "granted" ? ( + onRevokeProcessor(m) + : undefined + } + > + {t("users.cap.processor", "Processor")} + + ) : access !== "none" ? ( + // admin / team-owner role / inherited from a team-wide grant + + {t("users.cap.processor", "Processor")} + + ) : capabilities.manageGrants ? ( + onGrantProcessor(m)} + > + {t("users.cap.addProcessor", "+ Processor")} + + ) : null} + {showApprover && m.role === "admin" && ( + βœ“} + > + {t("users.cap.approver", "Approves policy")} + + )} +
+ + + {m.lastActive} + + + {capabilities.changeRole && ( +
+ { + setPending(e.target.value); + if (addError) setAddError(null); + }} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + add(); + } + }} + /> + +
+ )} + {addError &&

{addError}

} + + {value.length === 0 ? ( +

+ {emptyText ?? t("policies.labels.empty", "No labels yet.")} +

+ ) : grouped ? ( + + ) : ( +
+ {value.map(renderChip)} +
+ )} +
+ ); +} + +interface GroupedLabelsProps { + value: ClassificationLabel[]; + renderChip: (label: ClassificationLabel) => ReactNode; +} + +/** Chips laid out under collapsible parent categories (device-local structure). */ +function GroupedLabels({ value, renderChip }: GroupedLabelsProps) { + const { t } = useTranslation(); + const categories = useSyncExternalStore( + subscribeSidebarCategories, + getSidebarCategories, + ); + + // Category sections that actually contain labels from `value`, plus the leftovers. + const { sections, ungrouped } = useMemo(() => { + const byId = new Map(value.map((l) => [l.id, l])); + const claimed = new Set(); + const sections = categories + .map((category) => { + const members = category.labelKeys + .map((id) => byId.get(id)) + .filter((l): l is ClassificationLabel => !!l); + members.forEach((m) => claimed.add(m.id)); + return { category, members }; + }) + .filter((s) => s.members.length > 0); + const ungrouped = value.filter((l) => !claimed.has(l.id)); + return { sections, ungrouped }; + }, [value, categories]); + + // Only the first section starts expanded β€” a scannable overview. + const [expanded, setExpanded] = useState>( + () => new Set(sections.length > 0 ? [sections[0].category.id] : []), + ); + const toggle = (id: string) => + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + + return ( +
+ {sections.map(({ category, members }) => { + const isOpen = expanded.has(category.id); + return ( +
+ + {isOpen && ( +
+ {members.map(renderChip)} +
+ )} +
+ ); + })} + {ungrouped.length > 0 && ( +
+

+ {t("policies.labels.ungrouped", "Ungrouped")} +

+
+ {ungrouped.map(renderChip)} +
+
+ )} +
+ ); +} diff --git a/frontend/editor/src/proprietary/components/policies/LabelsEditorModal.tsx b/frontend/editor/src/proprietary/components/policies/LabelsEditorModal.tsx new file mode 100644 index 0000000000..9cafed3aeb --- /dev/null +++ b/frontend/editor/src/proprietary/components/policies/LabelsEditorModal.tsx @@ -0,0 +1,175 @@ +/** + * Full-screen ("fat") editor for the team's classification labels β€” the roomy + * view the settings summary's Edit button opens. Hosts the {@link LabelsEditor} + * chip grid plus an Import/Export toolbar and a footer holding the destructive + * actions (reset / start-from-scratch) and the Save/Cancel buttons. Editing is + * staged: nothing is persisted until Save, which stays disabled until the draft + * actually changes. The draft is owned by the caller so the settings summary + * reflects saved changes. + */ + +import { useRef } from "react"; +import { useTranslation } from "react-i18next"; +import FileDownloadOutlinedIcon from "@mui/icons-material/FileDownloadOutlined"; +import FileUploadOutlinedIcon from "@mui/icons-material/FileUploadOutlined"; +import RestartAltIcon from "@mui/icons-material/RestartAlt"; +import DeleteSweepOutlinedIcon from "@mui/icons-material/DeleteSweepOutlined"; +import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import { Modal } from "@app/ui/Modal"; +import { Button } from "@app/ui/Button"; +import { Banner } from "@app/ui/Banner"; +import { LabelsEditor } from "@app/components/policies/LabelsEditor"; +import type { ClassificationLabel } from "@app/data/classificationLabels"; + +interface LabelsEditorModalProps { + open: boolean; + onClose: () => void; + draft: ClassificationLabel[]; + onDraftChange: (next: ClassificationLabel[]) => void; + onImportFile: (file: File) => void; + onExport: () => void; + /** Stage the built-in default into the draft. */ + onReset: () => void; + /** Stage an empty list into the draft (build from scratch). */ + onClear: () => void; + onSave: () => void; + dirty: boolean; + saving: boolean; + readOnly: boolean; + /** Save (server) or import (file) failure to surface, if any. */ + error: string | null; +} + +export function LabelsEditorModal({ + open, + onClose, + draft, + onDraftChange, + onImportFile, + onExport, + onReset, + onClear, + onSave, + dirty, + saving, + readOnly, + error, +}: LabelsEditorModalProps) { + const { t } = useTranslation(); + const fileInput = useRef(null); + + const handleFile = (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + // Reset the input so picking the same file twice still fires onChange. + e.target.value = ""; + if (file) onImportFile(file); + }; + + return ( + +
+ {!readOnly && ( + <> + + + + )} +
+
+ + {!readOnly && ( + + )} +
+ + } + > +
+ {error && ( + } + description={error} + /> + )} + {!readOnly && ( +
+ + + +
+ )} + +
+
+ ); +} diff --git a/frontend/editor/src/proprietary/components/policies/Policies.css b/frontend/editor/src/proprietary/components/policies/Policies.css index 37b92a5830..feb698a644 100644 --- a/frontend/editor/src/proprietary/components/policies/Policies.css +++ b/frontend/editor/src/proprietary/components/policies/Policies.css @@ -90,19 +90,84 @@ outline: 2px solid var(--color-blue); outline-offset: -2px; } -/* Policy icons are colourless at rest; hovering or focusing the row reveals the - category colour (blue/purple/green/amber/red β€” see ROW_ACCENT). The accent - class sets --ib-base; we neutralise --ib-accent here and restore it on hover. */ -.pol-row .sui-iconbadge { - --ib-accent: var(--color-text-3); - transition: - color var(--motion-fast), - background var(--motion-fast); +/* Processing indicator: a spinning ring around the category icon while the + policy has runs in flight. */ +.pol-row-icon { + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + --pol-ring: var(--color-blue); } -.pol-row:hover .sui-iconbadge, -.pol-row:focus-visible .sui-iconbadge { +.pol-row-icon[data-accent="purple"] { + --pol-ring: var(--color-purple); +} +.pol-row-icon[data-accent="green"] { + --pol-ring: var(--color-green); +} +.pol-row-icon[data-accent="amber"] { + --pol-ring: var(--color-amber); +} +.pol-row-icon[data-accent="red"] { + --pol-ring: var(--color-red); +} +.pol-row-icon[data-accent="orange"] { + --pol-ring: var(--color-orange); +} +.pol-row-icon.is-processing .sui-iconbadge { --ib-accent: var(--ib-base); } +.pol-row-ring { + position: absolute; + inset: -3px; + border-radius: 999px; + border: 2px solid color-mix(in srgb, var(--pol-ring) 20%, transparent); + border-top-color: var(--pol-ring); + animation: pol-ring-spin 0.7s linear infinite; + pointer-events: none; +} + +/* Global "Retry failed policies (N)" action under the panel header β€” one place + to resume every file the chain stranded, whichever step failed. Amber, so it + reads as "needs attention" without screaming error. */ +.pol-retry-failed { + display: flex; + align-items: center; + gap: var(--space-1_5); + width: calc(100% - 2 * var(--space-1_5)); + margin: 0 var(--space-1_5) var(--space-1); + padding: var(--space-1) var(--space-2); + border: none; + border-radius: var(--radius-lg); + font-family: inherit; + font-size: 0.75rem; + font-weight: 500; + text-align: left; + cursor: pointer; + color: var(--color-amber, #d97706); + background: color-mix(in srgb, var(--color-amber, #d97706) 12%, transparent); + transition: background var(--motion-fast); +} +.pol-retry-failed:hover { + background: color-mix(in srgb, var(--color-amber, #d97706) 20%, transparent); +} +.pol-retry-failed:focus-visible { + outline: 2px solid var(--color-amber, #d97706); + outline-offset: -2px; +} +@keyframes pol-ring-spin { + to { + transform: rotate(360deg); + } +} + +@media (prefers-reduced-motion: reduce) { + .pol-row-ring { + animation-duration: 2s; + } +} + .pol-row-label { flex: 1; min-width: 0; @@ -110,6 +175,16 @@ font-weight: 500; color: var(--color-text-1); } + +.pol-row > .mantine-Button-inner { + width: 100%; +} +.pol-row > .mantine-Button-inner > .mantine-Button-label { + flex: 1; + min-width: 0; + justify-content: flex-start; + overflow: visible; +} .pol-row-trail { display: inline-flex; align-items: center; @@ -128,6 +203,81 @@ flex-shrink: 0; } +/* ── Policy settings: per-trigger run-order lists ── */ +.pol-reorder-section { + margin-top: var(--space-3); +} +.pol-reorder-list { + display: flex; + flex-direction: column; + margin-top: var(--space-1); +} +/* A reorder row: leading grip + tinted icon + label. Square (no radius) so the + drop line reads as one straight rule across the list. */ +.pol-reorder-row { + position: relative; + display: flex; + align-items: center; + gap: var(--space-2); + padding: var(--space-1_5) var(--space-2); +} +.pol-reorder-row[data-dragging] { + opacity: 0.4; +} +/* Straight, full-width blue insertion line at the drop position (no curves). */ +.pol-reorder-row[data-drop="above"]::before, +.pol-reorder-row[data-drop="below"]::after { + content: ""; + position: absolute; + left: 0; + right: 0; + height: 2px; + background: var(--color-blue); + pointer-events: none; +} +.pol-reorder-row[data-drop="above"]::before { + top: -1px; +} +.pol-reorder-row[data-drop="below"]::after { + bottom: -1px; +} +.pol-reorder-grip { + display: inline-flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + width: 1.25rem; + color: var(--color-text-4); + cursor: grab; +} +.pol-reorder-grip:active { + cursor: grabbing; +} +.pol-reorder-label { + flex: 1; + min-width: 0; + font-size: 0.8125rem; + font-weight: 500; + color: var(--color-text-1); +} +/* The drag ghost (a cloned row): the whole row with a full blue outline, kept + translucent so the list shows through as it moves. */ +.pol-reorder-row--ghost { + border-radius: var(--radius-lg); + outline: 2px solid var(--color-blue); + outline-offset: -2px; + background: var(--color-surface); + box-shadow: var(--shadow-md); + opacity: 0.55; +} +/* Empty-state line for a trigger with no policies. */ +.pol-reorder-empty { + margin: var(--space-1) 0 0; + padding: var(--space-1_5) var(--space-2); + font-size: 0.8125rem; + color: var(--color-text-3); +} + /* Retry button on a failed activity row. */ /* Expandable error text in the activity feed β€” long backend errors are clamped and collapsed by default so they don't blow up the row. */ @@ -278,6 +428,37 @@ margin: 0 0 var(--space-2); } +/* A section label rendered as a collapse toggle (Recent Activity): strip the + button chrome but keep the .pol-section-label typography, chevron pushed right. */ +.pol-section-toggle { + display: flex; + align-items: center; + gap: var(--space-2); + width: 100%; + background: none; + border: none; + padding: 0; + cursor: pointer; + text-align: left; + font-family: inherit; +} + +.pol-section-chevron { + margin-left: auto; + color: var(--color-text-4); + transition: transform 0.15s ease; +} +.pol-section-chevron.is-open { + transform: rotate(180deg); +} + +/* Recent-activity feed: cap to ~4.5 rows (and never more than ~45% of the + viewport) then scroll, so a long history doesn't push the stats footer away. */ +.pol-activity-list { + max-height: min(22rem, 45vh); + overflow-y: auto; +} + /* Sub-section header inside a settings card (e.g. "Output filename"). The field directly below it carries data-first so the borders don't double up. */ .pol-subhead { diff --git a/frontend/editor/src/proprietary/components/policies/PoliciesSidebar.tsx b/frontend/editor/src/proprietary/components/policies/PoliciesSidebar.tsx index 15b494c3f6..99d9e3169d 100644 --- a/frontend/editor/src/proprietary/components/policies/PoliciesSidebar.tsx +++ b/frontend/editor/src/proprietary/components/policies/PoliciesSidebar.tsx @@ -12,18 +12,39 @@ * collapsed; clicking an icon selects the policy and expands the rail. */ -import { useState, useEffect, useMemo, type ReactNode } from "react"; +import { + useState, + useEffect, + useMemo, + type DragEvent, + type ReactNode, +} from "react"; import { useTranslation } from "react-i18next"; +import { Menu } from "@mantine/core"; import ChevronRightIcon from "@mui/icons-material/ChevronRight"; -import LocalIcon from "@app/components/shared/LocalIcon"; +import ReplayRounded from "@mui/icons-material/ReplayRounded"; +import DragIndicatorRounded from "@mui/icons-material/DragIndicatorRounded"; +import MoreHorizRounded from "@mui/icons-material/MoreHorizRounded"; +import TuneRounded from "@mui/icons-material/TuneRounded"; +import InfoOutlined from "@mui/icons-material/InfoOutlined"; import { usePolicies } from "@app/hooks/usePolicies"; import { usePolicyCatalog } from "@app/hooks/usePolicyCatalog"; import { useAppConfig } from "@app/contexts/AppConfigContext"; import { useAuth } from "@app/auth/UseSession"; import { getPolicyAutomation } from "@app/services/policyFolders"; import { watchedFolderStorage } from "@app/services/watchedFolderStorage"; -import { runsToActivity, runsToStats } from "@app/services/policyLiveData"; -import { usePolicyRuns } from "@app/components/policies/policyRunStore"; +import { + runsToActivity, + runsToStats, + progressByCategory, + retryableFailedRuns, + EMPTY_RUN_PROGRESS, +} from "@app/services/policyLiveData"; +import { + removeRun, + usePolicyRuns, + usePolicyWaveStart, +} from "@app/components/policies/policyRunStore"; import { runPolicyOnFile } from "@app/components/policies/usePolicyAutoRun"; import type { FileId } from "@app/types/file"; import type { @@ -46,12 +67,15 @@ import { SectionHeader } from "@app/ui/SectionHeader"; import { PolicySetupWizard } from "@app/components/policies/PolicySetupWizard"; import { PolicyDetailPanel } from "@app/components/policies/PolicyDetailPanel"; import { PolicyDeleteConfirmModal } from "@app/components/policies/PolicyDeleteConfirmModal"; -import type { PolicyConfigResult } from "@app/types/policies"; +import type { PolicyCategory, PolicyConfigResult } from "@app/types/policies"; +import { PanelHeader } from "@app/ui/PanelHeader"; import { usePolicySelection, selectPolicy, setPolicyDetailView, closePolicy, + openPolicySettings, + closePolicySettings, } from "@app/components/policies/policySelectionStore"; import "@app/components/policies/Policies.css"; @@ -97,8 +121,8 @@ function promptGuestSignup(): void { * place of the tool list. False when the feature is off or nothing is selected. */ export function usePolicyDetailActive(): boolean { - const { selectedId } = usePolicySelection(); - return POLICIES_ENABLED && selectedId != null; + const { selectedId, settingsOpen } = usePolicySelection(); + return POLICIES_ENABLED && (selectedId != null || settingsOpen); } /** The collapsible policy list, rendered above the Tools section. */ @@ -113,6 +137,39 @@ export function PoliciesSection({ const pol = usePolicies(); const { categories } = usePolicyCatalog(); const guestBlocked = usePolicyGuestBlocked(); + + // Live run tallies drive the per-row processing ring + the header summary, + // scoped to the current upload wave so they don't accumulate across the whole + // persisted run history. + const runs = usePolicyRuns(); + const waveStart = usePolicyWaveStart(); + const progress = useMemo( + () => progressByCategory(runs, waveStart), + [runs, waveStart], + ); + // Failed runs still worth retrying, across ALL policies β€” drives the single + // global "Retry failed policies" action. Deliberately not per-policy: a chain + // failure strands the file mid-pipeline whichever step failed, and retrying + // globally resumes every stranded file (successful steps are never re-run; + // completed runs chain onward automatically). NOT wave-scoped: failures from + // earlier uploads still count. + const retryableRuns = useMemo(() => retryableFailedRuns(runs), [runs]); + const retryAllFailed = () => { + // Same replace-in-place pattern as the queue-full auto-retry: drop the stale + // failed row, fire a fresh run. Queue-full rejections during the burst are + // absorbed by the existing backoff. + for (const failed of retryableRuns) { + const backendId = pol.policies[failed.categoryId]?.backendId; + if (!backendId) continue; + removeRun(failed.runId); + void runPolicyOnFile( + failed.categoryId, + backendId, + failed.fileId as FileId, + failed.fileName, + ); + } + }; // Persist the expand/collapse state across refreshes. const [expanded, setExpanded] = useState(() => { try { @@ -148,6 +205,13 @@ export function PoliciesSection({ (c) => pol.policies[c.id]?.configured, ).length; + // Rows render in execution order (defaults to catalog order until reordered + // on the Policy settings page). + const displayCategories = [...visibleCategories].sort( + (a, b) => + (pol.policies[a.id]?.order ?? 0) - (pol.policies[b.id]?.order ?? 0), + ); + return (
@@ -161,37 +225,70 @@ export function PoliciesSection({ expanded={expanded} onToggle={toggleExpanded} /> - - + + + + + + + {/* Hovering surfaces the same explanation the info tooltip used to show. */} + + } + > + {t("policies.sidebar.whatIsPolicy", "What is a policy?")} + + + {pol.canConfigure && ( + } + onClick={() => openPolicySettings()} + > + {t("policies.sidebar.policySettings", "Policy settings")} + )} - > - - - + +
+ {retryableRuns.length > 0 && !guestBlocked && ( + + )} + {expanded && ( <>
- {visibleCategories.map((cat) => { + {displayCategories.map((cat) => { if (cat.comingSoon) { return (
@@ -203,8 +300,9 @@ export function PoliciesSection({ ); })} @@ -269,11 +402,25 @@ export function PoliciesSection({ ); } +/** + * Takeover dispatcher: shows the policy-settings page (execution order), an open + * policy's detail, or nothing β€” whichever the selection store currently holds. + * The heavy per-policy hooks live in {@link PolicyOpenDetail}, so the settings + * page doesn't pay for (or trip over) them. + */ +export function PolicyDetailTakeover() { + const { selectedId, settingsOpen } = usePolicySelection(); + if (!POLICIES_ENABLED) return null; + if (settingsOpen && selectedId == null) return ; + if (selectedId == null) return null; + return ; +} + /** * The open-policy view β€” narrative detail, setup wizard, or edit-settings β€” * which replaces the Tools area while a policy is selected. */ -export function PolicyDetailTakeover() { +function PolicyOpenDetail() { const { t } = useTranslation(); const pol = usePolicies(); const { categories, configs, sources, docTypes } = usePolicyCatalog(); @@ -470,6 +617,232 @@ export function PolicyDetailTakeover() { ); } +/** + * The Policy settings takeover β€” reached from the section header's "…" menu. + * Each trigger (upload / export) gets its own run-order list, since a chain only + * spans policies that fire on the same trigger β€” reordering one never affects the + * other. Admin-only (the menu entry is gated on canConfigure). + */ +function PolicySettingsPanel() { + const { t } = useTranslation(); + const pol = usePolicies(); + const { categories } = usePolicyCatalog(); + + // Configured, live policies for a trigger, in execution order. + const inOrder = (trigger: "upload" | "export") => + categories + .filter( + (c) => + pol.policies[c.id]?.configured && + !c.comingSoon && + (pol.policies[c.id]?.runOn ?? "upload") === trigger, + ) + .sort( + (a, b) => + (pol.policies[a.id]?.order ?? 0) - (pol.policies[b.id]?.order ?? 0), + ); + + const uploadCats = inOrder("upload"); + const exportCats = inOrder("export"); + + // Order is one global sort key, so persist both groups together (upload first) + // to keep each group's members contiguous β€” the auto-run chain reads relative + // order within a trigger. + const persist = (uploadIds: string[], exportIds: string[]) => + pol.reorderPolicies([...uploadIds, ...exportIds]); + + return ( +
+ } + title={t("policies.settings.title", "Policy settings")} + onClose={() => closePolicySettings()} + closeLabel={t("policies.detail.close", "Close")} + /> +
+

+ {t( + "policies.settings.runOrderDesc", + "When more than one policy runs on the same trigger, they run in this order β€” each on the previous policy's output. Drag to reorder.", + )} +

+ {/* Both triggers are always shown so the run order for each is explicit, + with an empty note when a trigger has no policies. */} + + persist( + ids, + exportCats.map((c) => c.id), + ) + } + /> + + persist( + uploadCats.map((c) => c.id), + ids, + ) + } + /> +
+
+ ); +} + +/** + * One trigger's run-order list. Rows drag to reorder (only when there's more than + * one to order); the drag ghost is the whole row with a blue outline, and a + * straight blue line marks where the policy will land. Reorders in isolation and + * hands the new id order back to the parent to persist. + */ +function PolicyReorderSection({ + title, + cats, + emptyText, + onReorder, +}: { + title: string; + cats: PolicyCategory[]; + emptyText: string; + onReorder: (orderedIds: string[]) => void; +}) { + const { t } = useTranslation(); + const [dragId, setDragId] = useState(null); + const [overId, setOverId] = useState(null); + // Whether the drop would land after (vs before) the hovered row. + const [overBelow, setOverBelow] = useState(false); + const draggable = cats.length >= 2; + + const clear = () => { + setDragId(null); + setOverId(null); + }; + + const handleDrop = (targetId: string) => { + if (!dragId || dragId === targetId) return clear(); + const ids = cats.map((c) => c.id); + const from = ids.indexOf(dragId); + let to = ids.indexOf(targetId) + (overBelow ? 1 : 0); + if (from < 0 || to < 0) return clear(); + ids.splice(from, 1); + if (from < to) to -= 1; + ids.splice(to, 0, dragId); + onReorder(ids); + clear(); + }; + + // The native drag image would be just the grip under the cursor; instead snapshot + // the whole row (a styled clone) so the ghost that follows the mouse is the full + // row with a blue outline. + const startDrag = (e: DragEvent, catId: string) => { + setDragId(catId); + e.dataTransfer.effectAllowed = "move"; + const row = (e.currentTarget as HTMLElement).closest(".pol-reorder-row"); + if (row instanceof HTMLElement) { + const clone = row.cloneNode(true) as HTMLElement; + clone.classList.add("pol-reorder-row--ghost"); + clone.style.width = `${row.offsetWidth}px`; + clone.style.position = "fixed"; + clone.style.top = "-1000px"; + clone.style.left = "-1000px"; + clone.style.pointerEvents = "none"; + document.body.appendChild(clone); + e.dataTransfer.setDragImage(clone, 24, row.offsetHeight / 2); + window.setTimeout(() => clone.remove(), 0); + } + }; + + if (cats.length === 0) { + return ( +
+

{title}

+

{emptyText}

+
+ ); + } + + return ( +
+

{title}

+
+ {cats.map((cat) => ( +
{ + if (!dragId) return; + e.preventDefault(); + const rect = e.currentTarget.getBoundingClientRect(); + setOverId(cat.id); + setOverBelow(e.clientY > rect.top + rect.height / 2); + } + : undefined + } + onDragLeave={ + draggable + ? () => setOverId((id) => (id === cat.id ? null : id)) + : undefined + } + onDrop={ + draggable + ? (e) => { + e.preventDefault(); + handleDrop(cat.id); + } + : undefined + } + > + {draggable && ( + startDrag(e, cat.id)} + onDragEnd={clear} + role="button" + tabIndex={-1} + aria-label={t( + "policies.settings.reorderHandle", + "Drag to reorder", + )} + > + + + )} + + {cat.icon} + + + {t(`policies.catalog.${cat.id}`, cat.label)} + +
+ ))} +
+
+ ); +} + /** * Collapsed-rail policy icons. Each tints blue when active and carries a small * status dot (green active / amber paused). Clicking selects the policy and diff --git a/frontend/editor/src/proprietary/components/policies/PolicyDetailPanel.tsx b/frontend/editor/src/proprietary/components/policies/PolicyDetailPanel.tsx index 4afcf5891c..95f554edcc 100644 --- a/frontend/editor/src/proprietary/components/policies/PolicyDetailPanel.tsx +++ b/frontend/editor/src/proprietary/components/policies/PolicyDetailPanel.tsx @@ -7,6 +7,7 @@ import DescriptionIcon from "@mui/icons-material/Description"; import CheckCircleIcon from "@mui/icons-material/CheckCircle"; import WarningAmberIcon from "@mui/icons-material/WarningAmber"; import AutorenewIcon from "@mui/icons-material/Autorenew"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; import LockIcon from "@mui/icons-material/Lock"; import DeleteOutlineIcon from "@mui/icons-material/DeleteOutlined"; import { PanelHeader } from "@app/ui/PanelHeader"; @@ -115,6 +116,7 @@ export function PolicyDetailPanel({ }: PolicyDetailPanelProps) { const { t } = useTranslation(); const isPaused = status === "paused"; + const [activityOpen, setActivityOpen] = useState(true); // Real configured steps drive the flow; fall back to the preset's rule labels. const enforceItems = steps && steps.length > 0 @@ -181,70 +183,91 @@ export function PolicyDetailPanel({ {/* Recent Activity */}
-

- {t("policies.detail.recentActivity", "Recent Activity")} -

- {activityItems.length > 0 ? ( - - {activityItems.map((item, i) => ( - 0} - leadingTone={ - item.status === "flagged" - ? "warning" - : item.status === "processing" - ? "info" - : "success" - } - leading={ - item.status === "flagged" ? ( - - ) : item.status === "processing" ? ( - - ) : ( - - ) - } - title={item.doc} - description={ - item.status === "flagged" ? ( - - ) : ( - item.action - ) - } - meta={item.time} - trailing={ - item.status === "flagged" && onRetry ? ( - - ) : undefined - } - /> - ))} - - ) : ( - - } - title={t("policies.detail.noActivityTitle", "No activity yet")} - description={t( - "policies.detail.noActivityDescription", - "Documents will appear here once this policy runs.", - )} + + {activityOpen && + (activityItems.length > 0 ? ( + +
+ {activityItems.map((item, i) => ( + 0} + leadingTone={ + item.status === "flagged" + ? "warning" + : item.status === "processing" + ? "info" + : "success" + } + leading={ + item.status === "flagged" ? ( + + ) : item.status === "processing" ? ( + + ) : ( + + ) + } + title={item.doc} + description={ + item.status === "flagged" ? ( + + ) : ( + item.action + ) + } + meta={item.time} + trailing={ + item.status === "flagged" && onRetry ? ( + + ) : undefined + } + /> + ))} +
+
+ ) : ( + + } + title={t( + "policies.detail.noActivityTitle", + "No activity yet", + )} + description={t( + "policies.detail.noActivityDescription", + "Documents will appear here once this policy runs.", + )} + /> + + ))}
{/* Stats β€” one grouped card with divided columns, intentionally diff --git a/frontend/editor/src/proprietary/components/policies/PolicyPiiField.tsx b/frontend/editor/src/proprietary/components/policies/PolicyPiiField.tsx index d9fef2361d..59fe76c021 100644 --- a/frontend/editor/src/proprietary/components/policies/PolicyPiiField.tsx +++ b/frontend/editor/src/proprietary/components/policies/PolicyPiiField.tsx @@ -1,4 +1,4 @@ -import { MultiSelect } from "@mantine/core"; +import { MultiSelect } from "@app/ui/MultiSelect"; import { useTranslation } from "react-i18next"; import { PII_PRESETS } from "@app/data/policyDefinitions"; @@ -49,8 +49,8 @@ export function PolicyPiiField({ return ( ({ value: p.value, @@ -62,7 +62,6 @@ export function PolicyPiiField({ onChange={handleChange} disabled={disabled} clearable - checkIconPosition="right" /> ); } diff --git a/frontend/editor/src/proprietary/components/policies/PolicySetupWizard.tsx b/frontend/editor/src/proprietary/components/policies/PolicySetupWizard.tsx index d096b8327e..cffb02c28c 100644 --- a/frontend/editor/src/proprietary/components/policies/PolicySetupWizard.tsx +++ b/frontend/editor/src/proprietary/components/policies/PolicySetupWizard.tsx @@ -36,6 +36,7 @@ import { } from "@app/components/policies/PolicyWorkflowStep"; import { PolicyToolConfigStep } from "@app/components/policies/PolicyToolConfigStep"; import { getPolicyToolChain } from "@app/components/policies/policyToolChains"; +import { ClassificationLabelsSection } from "@app/components/policies/ClassificationLabelsSection"; // Sources are always "editor" for this release, so the Sources step is dropped // from the flow (its panel code is kept below for when other sources return). @@ -103,6 +104,9 @@ export function PolicySetupWizard({ // Preset (tool-chain) policies render the locked tool config as their Workflow // step instead of the add/remove builder. const toolChain = getPolicyToolChain(category.id); + // A single-tool chain has nothing to toggle/configure, so its config UI is + // hidden (kept mounted so the submit trigger still emits that one tool). + const singleToolChain = toolChain != null && toolChain.length === 1; const { user } = useAuth(); const [step, setStep] = useState(1); const [fieldValues, setFieldValues] = useState(() => @@ -337,22 +341,29 @@ export function PolicySetupWizard({
{toolChain ? ( <> -

- {t( - "policies.wizard.toolChainDesc", - "Configure the tools this policy runs on each document.", - )} -

- + {/* Single-tool chains have nothing to configure β€” hide the prompt + and the toggle, but keep the step mounted (display:none) so the + final submit still emits that one tool. */} + {!singleToolChain && ( +

+ {t( + "policies.wizard.toolChainDesc", + "Configure the tools this policy runs on each document.", + )} +

+ )} +
+ +
) : ( <> @@ -371,6 +382,12 @@ export function PolicySetupWizard({ /> )} + {/* The Classification policy owns the editable label sets (team-shared + + personal) the classifier picks from. Kept on the first step + alongside the tool so it's not buried. */} + {category.id === "classification" && ( + + )}
{step === 2 && ( @@ -679,7 +696,7 @@ export function PolicySetupWizard({ : [...prev, dt], ) } - label={dt} + label={t(`policies.docType.${dt}`, dt)} /> ))}
diff --git a/frontend/editor/src/proprietary/components/policies/PolicyWatermarkConfig.tsx b/frontend/editor/src/proprietary/components/policies/PolicyWatermarkConfig.tsx index 692508a92e..8697c9d030 100644 --- a/frontend/editor/src/proprietary/components/policies/PolicyWatermarkConfig.tsx +++ b/frontend/editor/src/proprietary/components/policies/PolicyWatermarkConfig.tsx @@ -9,10 +9,10 @@ interface PolicyWatermarkConfigProps { } /** - * Watermark configuration for a policy: the full watermark settings minus the - * "Flatten PDF pages to images" toggle (hidden), with flatten forced on so the - * watermark is baked into the page and can't be stripped out. Normalised once - * on mount. + * Watermark configuration for a policy: text watermarks only (the type selector + * and image option are hidden), minus the "Flatten PDF pages to images" toggle + * (hidden), with flatten forced on so the watermark is baked into the page and + * can't be stripped out. Normalised once on mount. */ export function PolicyWatermarkConfig({ parameters, @@ -20,9 +20,11 @@ export function PolicyWatermarkConfig({ disabled, }: PolicyWatermarkConfigProps) { useEffect(() => { - if (parameters.convertPDFToImage !== true) { - onChange({ ...parameters, convertPDFToImage: true }); - } + const patch: Record = {}; + if (parameters.convertPDFToImage !== true) patch.convertPDFToImage = true; + // Policies only support text watermarks. + if (parameters.watermarkType !== "text") patch.watermarkType = "text"; + if (Object.keys(patch).length > 0) onChange({ ...parameters, ...patch }); }, []); return ( @@ -33,6 +35,7 @@ export function PolicyWatermarkConfig({ } disabled={disabled} showFlatten={false} + textOnly /> ); } diff --git a/frontend/editor/src/proprietary/components/policies/policyRunStore.test.ts b/frontend/editor/src/proprietary/components/policies/policyRunStore.test.ts index 2034866b87..498a688ac2 100644 --- a/frontend/editor/src/proprietary/components/policies/policyRunStore.test.ts +++ b/frontend/editor/src/proprietary/components/policies/policyRunStore.test.ts @@ -83,12 +83,81 @@ describe("policyRunStore", () => { expect(isDispatched("security", "f1")).toBe(true); }); - it("caps stored runs at 50, newest first", () => { - for (let i = 0; i < 55; i++) { - recordRunStart(rec({ runId: `r${i}`, fileId: `f${i}`, startedAt: i })); + it("never evicts in-flight runs, even past the soft cap", () => { + // A large upload batch can exceed the cap while still processing. Dropping a + // live run would orphan its polling/import and undercount progress, so every + // in-flight run is kept regardless of the cap. + for (let i = 0; i < 210; i++) { + recordRunStart( + rec({ + runId: `r${i}`, + fileId: `f${i}`, + status: "PENDING", + startedAt: i, + }), + ); } const runs = read("stirling-policy-runs").runs; - expect(runs).toHaveLength(50); - expect(runs[0].runId).toBe("r54"); // most recent + expect(runs).toHaveLength(210); + expect(runs[0].runId).toBe("r209"); // newest first + }); + + it("evicts the oldest TERMINAL runs first once over the cap", () => { + // Imported, since a COMPLETED-but-not-yet-imported run still counts as + // in-flight (see isRunInFlight) and must never be evicted. + for (let i = 0; i < 210; i++) { + recordRunStart( + rec({ + runId: `r${i}`, + fileId: `f${i}`, + status: "COMPLETED", + imported: true, + startedAt: i, + }), + ); + } + const runs = read("stirling-policy-runs").runs; + expect(runs).toHaveLength(200); // trimmed to MAX_RUNS + expect(runs[0].runId).toBe("r209"); // newest kept + expect(runs.some((r: PolicyRunRecord) => r.runId === "r0")).toBe(false); // oldest dropped + }); + + it("does not evict a COMPLETED run that hasn't been imported yet, even past the cap", () => { + for (let i = 0; i < 210; i++) { + recordRunStart( + rec({ + runId: `r${i}`, + fileId: `f${i}`, + status: "COMPLETED", + startedAt: i, + }), + ); + } + const runs = read("stirling-policy-runs").runs; + expect(runs).toHaveLength(210); + expect(runs.some((r: PolicyRunRecord) => r.runId === "r0")).toBe(true); + }); + + describe("processing wave (scopes the panel's progress counts to this upload)", () => { + it("begins a new wave when recording with nothing in flight", () => { + recordRunStart(rec({ runId: "a", fileId: "fa", startedAt: 500 })); + expect(read("stirling-policy-runs").waveStartedAt).toBe(500); + }); + + it("keeps the wave while earlier runs are still in flight", () => { + recordRunStart(rec({ runId: "a", fileId: "fa", startedAt: 100 })); + recordRunStart(rec({ runId: "b", fileId: "fb", startedAt: 200 })); + // b joined a's wave (a still PENDING) β€” the boundary stays at a. + expect(read("stirling-policy-runs").waveStartedAt).toBe(100); + }); + + it("starts a fresh wave once the prior batch has all finished", () => { + recordRunStart(rec({ runId: "a", fileId: "fa", startedAt: 100 })); + // Prior batch completes AND imports β†’ no longer in flight. + updateRun("a", { status: "COMPLETED", imported: true }); + // A new upload after the lull resets the wave to itself. + recordRunStart(rec({ runId: "b", fileId: "fb", startedAt: 5000 })); + expect(read("stirling-policy-runs").waveStartedAt).toBe(5000); + }); }); }); diff --git a/frontend/editor/src/proprietary/components/policies/policyRunStore.ts b/frontend/editor/src/proprietary/components/policies/policyRunStore.ts index 9485df3a9e..2fd91d215a 100644 --- a/frontend/editor/src/proprietary/components/policies/policyRunStore.ts +++ b/frontend/editor/src/proprietary/components/policies/policyRunStore.ts @@ -59,11 +59,56 @@ export const POLICY_IN_FLIGHT_STATUSES: readonly PolicyRunStatus[] = [ interface RunState { runs: PolicyRunRecord[]; dispatched: string[]; + /** startedAt of the run that began the current processing "wave" β€” a burst of + * runs with no idle gap. Reset whenever a run is recorded while nothing is in + * flight. The panel's progress counts (X of Y processed) scope to this so they + * reflect the CURRENT upload, not the whole persisted history. */ + waveStartedAt: number; +} + +const IN_FLIGHT_STATUSES: ReadonlySet = new Set([ + "PENDING", + "RUNNING", + "WAITING_FOR_INPUT", +]); + +/** True while a run is still working: dispatched/running, in a retry backoff, or + * done on the backend but not yet imported into the workspace. Shared so the + * wave tracking, the panel progress, and the file-row spinner all agree. */ +export function isRunInFlight(run: PolicyRunRecord): boolean { + if (run.retrying) return true; + if (IN_FLIGHT_STATUSES.has(run.status)) return true; + return run.status === "COMPLETED" && !run.imported; } const STORAGE_KEY = "stirling-policy-runs"; -/** Cap stored runs so the activity log can't grow without bound. */ -const MAX_RUNS = 50; +/** Soft cap on stored runs so the activity log can't grow without bound. Only + * TERMINAL runs are ever evicted (see {@link capRuns}); in-flight runs are always + * kept, so a large upload batch (more files than this cap) still polls + imports + * every one β€” and the panel's processing count stays accurate. */ +const MAX_RUNS = 200; + +const TERMINAL: ReadonlySet = new Set([ + "COMPLETED", + "FAILED", + "CANCELLED", +]); + +/** + * Trim the run list toward MAX_RUNS by dropping the OLDEST terminal runs first, + * never in-flight ones. Runs are newest-first, so we walk from the tail. If more + * than MAX_RUNS runs are still in flight (a very large batch) they're all kept β€” + * dropping a live run would orphan its polling/import and undercount progress. + */ +function capRuns(runs: PolicyRunRecord[]): PolicyRunRecord[] { + if (runs.length <= MAX_RUNS) return runs; + const trimmed = [...runs]; + for (let i = trimmed.length - 1; i >= 0 && trimmed.length > MAX_RUNS; i--) { + const run = trimmed[i]; + if (TERMINAL.has(run.status) && !isRunInFlight(run)) trimmed.splice(i, 1); + } + return trimmed; +} function read(): RunState { try { @@ -88,12 +133,14 @@ function read(): RunState { })) : [], dispatched: Array.isArray(parsed.dispatched) ? parsed.dispatched : [], + waveStartedAt: + typeof parsed.waveStartedAt === "number" ? parsed.waveStartedAt : 0, }; } } catch { // Corrupt/unavailable storage β€” start empty. } - return { runs: [], dispatched: [] }; + return { runs: [], dispatched: [], waveStartedAt: 0 }; } let state: RunState = read(); @@ -134,11 +181,22 @@ function getSnapshot(): RunState { return state; } -const SERVER_SNAPSHOT: RunState = { runs: [], dispatched: [] }; +const SERVER_SNAPSHOT: RunState = { + runs: [], + dispatched: [], + waveStartedAt: 0, +}; function getServerSnapshot(): RunState { return SERVER_SNAPSHOT; } +/** Non-hook check for background work that should yield while a policy wave is + * running (e.g. the sidebar's category backfill) β€” reads the store without + * subscribing, so callers don't re-render on every poll tick. */ +export function hasInFlightPolicyRuns(): boolean { + return state.runs.some(isRunInFlight); +} + /** Key identifying a single (policy, file) run attempt. */ export function dispatchKey(categoryId: string, fileId: string): string { return `${categoryId}:${fileId}`; @@ -152,11 +210,18 @@ export function isDispatched(categoryId: string, fileId: string): boolean { /** Record a newly-dispatched run (marks it dispatched + adds the record). */ export function recordRunStart(record: PolicyRunRecord) { const key = dispatchKey(record.categoryId, record.fileId); + // A run recorded while nothing else is in flight begins a fresh wave, so the + // progress counts reset to this upload instead of accumulating across every + // past upload persisted in localStorage. + const waveStartedAt = state.runs.some(isRunInFlight) + ? state.waveStartedAt + : record.startedAt; state = { - runs: [record, ...state.runs].slice(0, MAX_RUNS), + runs: capRuns([record, ...state.runs]), dispatched: state.dispatched.includes(key) ? state.dispatched : [...state.dispatched, key], + waveStartedAt, }; emit(); } @@ -169,7 +234,7 @@ export function recordRunStart(record: PolicyRunRecord) { */ export function addReconciledRun(record: PolicyRunRecord) { if (state.runs.some((r) => r.runId === record.runId)) return; - state = { ...state, runs: [record, ...state.runs].slice(0, MAX_RUNS) }; + state = { ...state, runs: capRuns([record, ...state.runs]) }; emit(); } @@ -209,7 +274,7 @@ export function removeRun(runId: string) { /** Reset the store β€” used by tests to isolate it. */ export function resetPolicyRuns() { - state = { runs: [], dispatched: [] }; + state = { runs: [], dispatched: [], waveStartedAt: 0 }; emit(); } @@ -220,3 +285,13 @@ export function usePolicyRuns(): PolicyRunRecord[] { () => getServerSnapshot().runs, ); } + +/** startedAt of the current processing wave (see {@link RunState.waveStartedAt}). + * The panel scopes its progress counts to runs at/after this. */ +export function usePolicyWaveStart(): number { + return useSyncExternalStore( + subscribe, + () => getSnapshot().waveStartedAt, + () => getServerSnapshot().waveStartedAt, + ); +} diff --git a/frontend/editor/src/proprietary/components/policies/policySelectionStore.ts b/frontend/editor/src/proprietary/components/policies/policySelectionStore.ts index 451582b90e..e8b80c06c8 100644 --- a/frontend/editor/src/proprietary/components/policies/policySelectionStore.ts +++ b/frontend/editor/src/proprietary/components/policies/policySelectionStore.ts @@ -15,9 +15,16 @@ import type { PolicyDetailView } from "@app/types/policies"; interface PolicySelection { selectedId: string | null; detailView: PolicyDetailView; + /** The policy-settings page (execution order) takes over the rail. Independent + * of {@link selectedId} β€” it's a section-level view, not tied to one policy. */ + settingsOpen: boolean; } -let state: PolicySelection = { selectedId: null, detailView: "detail" }; +let state: PolicySelection = { + selectedId: null, + detailView: "detail", + settingsOpen: false, +}; const listeners = new Set<() => void>(); function emit() { @@ -37,6 +44,7 @@ function getSnapshot(): PolicySelection { const SERVER_SNAPSHOT: PolicySelection = { selectedId: null, detailView: "detail", + settingsOpen: false, }; function getServerSnapshot(): PolicySelection { return SERVER_SNAPSHOT; @@ -44,7 +52,7 @@ function getServerSnapshot(): PolicySelection { /** Open a policy's detail (resets the sub-view to the narrative). */ export function selectPolicy(id: string | null) { - state = { selectedId: id, detailView: "detail" }; + state = { selectedId: id, detailView: "detail", settingsOpen: false }; emit(); } @@ -55,6 +63,19 @@ export function setPolicyDetailView(view: PolicyDetailView) { emit(); } +/** Open the policy-settings page (execution order). Clears any open policy. */ +export function openPolicySettings() { + state = { selectedId: null, detailView: "detail", settingsOpen: true }; + emit(); +} + +/** Close the policy-settings page and return to the list. */ +export function closePolicySettings() { + if (!state.settingsOpen) return; + state = { ...state, settingsOpen: false }; + emit(); +} + /** Close the open policy and return to the list. */ export function closePolicy() { selectPolicy(null); @@ -62,7 +83,7 @@ export function closePolicy() { /** Reset to the initial state β€” used by tests to isolate the module store. */ export function resetPolicySelection() { - state = { selectedId: null, detailView: "detail" }; + state = { selectedId: null, detailView: "detail", settingsOpen: false }; emit(); } diff --git a/frontend/editor/src/proprietary/components/policies/policyStatus.ts b/frontend/editor/src/proprietary/components/policies/policyStatus.ts index e45d280654..ada31ad11b 100644 --- a/frontend/editor/src/proprietary/components/policies/policyStatus.ts +++ b/frontend/editor/src/proprietary/components/policies/policyStatus.ts @@ -17,27 +17,31 @@ export const STATUS_LABEL: Record = { setup: "Set up", }; -/** - * Per-category accent colour - */ +/** Per-category icon accent β€” neutral (no tint background) across all categories. */ export const ROW_ACCENT: Record = { + ingestion: "neutral", + security: "neutral", + compliance: "neutral", + routing: "neutral", + retention: "neutral", +}; + +/** Per-category colour for the file badges + enforcement overlay. Separate from + * ROW_ACCENT: the sidebar rows render neutral by design, but the badges keep + * their identity colours so files remain distinguishable at a glance. */ +const BADGE_ACCENT: Record = { ingestion: "blue", + classification: "orange", security: "purple", compliance: "green", routing: "amber", retention: "red", }; -/** Accent name β†’ the CSS colour var the policy badges tint with. */ -const ACCENT_VAR: Record = { - blue: "var(--color-blue)", - purple: "var(--color-purple)", - green: "var(--color-green)", - amber: "var(--color-amber)", - red: "var(--color-red)", -}; - -/** CSS colour var for a policy category's accent (blue for unknown categories). */ +/** + * CSS colour var for a policy category's badge accent (blue for unknown + * categories) β€” the tint used by the file badges and the enforcement overlay. + */ export function policyAccentVar(categoryId: string): string { - return ACCENT_VAR[ROW_ACCENT[categoryId] ?? "blue"]; + return `var(--color-${BADGE_ACCENT[categoryId] ?? "blue"})`; } diff --git a/frontend/editor/src/proprietary/components/policies/policyToolChains.ts b/frontend/editor/src/proprietary/components/policies/policyToolChains.ts index e3cd231ea1..a7c224d1ea 100644 --- a/frontend/editor/src/proprietary/components/policies/policyToolChains.ts +++ b/frontend/editor/src/proprietary/components/policies/policyToolChains.ts @@ -8,6 +8,9 @@ export const POLICY_TOOL_CHAINS: Record = { // Security: redact PII + watermark + sanitize (strips JS). Which are enabled // by default comes from the preset's defaultOperations, not this list. security: ["redact", "watermark", "sanitize"], + // Classification: a single backend step that classifies the document and + // writes the result into its metadata. + classification: ["classify"], }; /** The configurable tool chain for a category, or null if it has none yet. */ diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx new file mode 100644 index 0000000000..1d1bec13e6 --- /dev/null +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx @@ -0,0 +1,279 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderHook, act } from "@testing-library/react"; + +/** + * Batch integration test for the policy auto-run orchestration, at the scale the + * user hit the bug: 61 files uploaded at once, two active upload policies + * (Classification β†’ Security) chained. Drives the REAL policyRunStore + the REAL + * hook effects (dispatch β†’ poll β†’ import β†’ chain), mocking only the IO boundaries + * (network, storage, thumbnail/stub creation). + * + * Proves the invariants the user asked for: + * - 61 files β‡’ exactly 122 runs (61 classification, then 61 security). + * - Delivery is SILENT + in place (consumeFiles called with { silent: true }), + * never adding a second copy β€” the workspace never grows past 61. + * - No runaway: if the loop guard regressed, the run count would blow past 122 + * (or the test would time out), so an exact 122 is a hard regression gate. + * - Closing all files mid-run does NOT re-open them: with the workspace emptied, + * outputs are delivered to storage (persistVersionedOutputs), never re-added + * to the workspace via consumeFiles. + */ + +const FILE_COUNT = 61; + +// A tiny mutable "workspace" the mocks share: the list of file stubs currently in +// the workbench, mirrored into useAllFiles. consumeFiles mutates it in place +// (input id β†’ output id) exactly as the real silent reducer would. +const mocks = vi.hoisted(() => ({ + workspace: [] as Array<{ id: string }>, + consumeSilentCalls: 0, + consumeNonSilentCalls: 0, + persistCalls: 0, + addFilesCalls: 0, + stubCounter: 0, + backendOutCounter: 0, + bumpRevision: vi.fn(), + runStoredPolicy: vi.fn(), + getPolicyRun: vi.fn(), + listPolicyRuns: vi.fn(), + downloadPolicyOutput: vi.fn(), + getStirlingFile: vi.fn(), + getStirlingFileStub: vi.fn(), + persistVersionedOutputs: vi.fn(), + updateFileMetadata: vi.fn(), + createStirlingFilesAndStubs: vi.fn(), + addFiles: vi.fn(), + updateStirlingFileStub: vi.fn(), + consumeFiles: vi.fn(), +})); + +vi.mock("@app/constants/featureFlags", () => ({ POLICIES_ENABLED: true })); +vi.mock("@app/contexts/FileContext", () => ({ + useAllFiles: () => ({ fileStubs: mocks.workspace }), + useFileManagement: () => ({ + addFiles: mocks.addFiles, + updateStirlingFileStub: mocks.updateStirlingFileStub, + }), + useFileContext: () => ({ consumeFiles: mocks.consumeFiles }), +})); +vi.mock("@app/contexts/IndexedDBContext", () => ({ + useIndexedDB: () => ({ bumpRevision: mocks.bumpRevision }), +})); +vi.mock("@app/hooks/usePolicies", () => ({ + usePolicies: () => ({ + policies: { + // Classification runs first (order 0), Security second (order 1). + classification: { + configured: true, + status: "active", + backendId: "backend-classification", + runOn: "upload", + order: 0, + outputMode: "new_version", + outputName: "", + }, + security: { + configured: true, + status: "active", + backendId: "backend-security", + runOn: "upload", + order: 1, + outputMode: "new_version", + outputName: "", + }, + }, + }), +})); +vi.mock("@app/services/policyApi", () => ({ + runStoredPolicy: mocks.runStoredPolicy, + getPolicyRun: mocks.getPolicyRun, + listPolicyRuns: mocks.listPolicyRuns, + downloadPolicyOutput: mocks.downloadPolicyOutput, + resolvePolicyRunTarget: () => "saas", +})); +vi.mock("@app/services/fileStorage", () => ({ + fileStorage: { + getStirlingFile: mocks.getStirlingFile, + getStirlingFileStub: mocks.getStirlingFileStub, + persistVersionedOutputs: mocks.persistVersionedOutputs, + updateFileMetadata: mocks.updateFileMetadata, + }, +})); +vi.mock("@app/services/fileStubHelpers", () => ({ + createStirlingFilesAndStubs: mocks.createStirlingFilesAndStubs, +})); +vi.mock("@app/services/fileClassification", () => ({ + readClassificationLabelsFromFile: vi.fn().mockResolvedValue(null), +})); + +import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun"; +import { + usePolicyRuns, + resetPolicyRuns, +} from "@app/components/policies/policyRunStore"; +import type { PolicyRunRecord } from "@app/components/policies/policyRunStore"; + +/** A stable snapshot of the store, read after the flow settles. */ +let latestRuns: PolicyRunRecord[] = []; +function Harness() { + usePolicyAutoRun(); + latestRuns = usePolicyRuns(); + return null; +} + +function replaceInWorkspace(inputIds: string[], outputIds: string[]) { + mocks.workspace = mocks.workspace + .filter((s) => !inputIds.includes(s.id)) + .concat(outputIds.map((id) => ({ id }))); +} + +beforeEach(() => { + localStorage.clear(); + resetPolicyRuns(); + vi.clearAllMocks(); + mocks.consumeSilentCalls = 0; + mocks.consumeNonSilentCalls = 0; + mocks.persistCalls = 0; + mocks.addFilesCalls = 0; + mocks.stubCounter = 0; + mocks.backendOutCounter = 0; + + mocks.workspace = Array.from({ length: FILE_COUNT }, (_, i) => ({ + id: `file-${i}`, + })); + + mocks.listPolicyRuns.mockResolvedValue([]); + // A run's bytes are always resolvable (input files + versioned children). + mocks.getStirlingFile.mockResolvedValue( + new File(["x"], "doc.pdf", { type: "application/pdf" }), + ); + mocks.getStirlingFileStub.mockResolvedValue(null); + mocks.persistVersionedOutputs.mockImplementation(async () => { + mocks.persistCalls += 1; + }); + mocks.updateFileMetadata.mockResolvedValue(false); + mocks.downloadPolicyOutput.mockResolvedValue( + new Blob(["x"], { type: "application/pdf" }), + ); + + // Each dispatch gets a unique run id; the run's single backend output likewise. + mocks.runStoredPolicy.mockImplementation( + async () => `run-${mocks.stubCounter++}`, + ); + mocks.getPolicyRun.mockImplementation(async (runId: string) => ({ + runId, + policyId: null, + status: "COMPLETED", + currentStep: 1, + stepCount: 1, + error: null, + outputs: [ + { + fileId: `backend-out-${mocks.backendOutCounter++}`, + fileName: "doc.pdf", + }, + ], + })); + // Deliver a unique workspace child stub per output, derived from the parent so + // the chain's second policy can find + version it. + mocks.createStirlingFilesAndStubs.mockImplementation( + async (files: File[], parentStub: { id: string }) => { + const stubs = files.map(() => ({ + id: `${parentStub.id}~${mocks.stubCounter++}`, + })); + return { stirlingFiles: files, stubs }; + }, + ); + mocks.addFiles.mockImplementation(async (files: File[]) => { + mocks.addFilesCalls += 1; + return files.map((_f, i) => ({ + fileId: `added-${mocks.stubCounter++}-${i}`, + })); + }); + mocks.consumeFiles.mockImplementation( + async ( + inputIds: string[], + _outputs: unknown[], + stubs: Array<{ id: string }>, + options?: { silent?: boolean }, + ) => { + if (options?.silent) mocks.consumeSilentCalls += 1; + else mocks.consumeNonSilentCalls += 1; + replaceInWorkspace( + inputIds, + stubs.map((s) => s.id), + ); + }, + ); +}); + +/** Drive the hook until the store shows the expected number of imported runs. */ +async function runUntilSettled(expectedRuns: number) { + renderHook(() => Harness()); + await act(async () => { + await vi.waitFor( + () => { + const imported = latestRuns.filter((r) => r.imported).length; + expect(imported).toBe(expectedRuns); + }, + { timeout: 8000, interval: 20 }, + ); + }); +} + +describe("policy auto-run β€” 61-file batch through a Classification β†’ Security chain", () => { + it("produces exactly 122 runs (61 classification, then 61 security)", async () => { + await runUntilSettled(FILE_COUNT * 2); + + const classification = latestRuns.filter( + (r) => r.categoryId === "classification", + ); + const security = latestRuns.filter((r) => r.categoryId === "security"); + + expect(classification).toHaveLength(FILE_COUNT); + expect(security).toHaveLength(FILE_COUNT); + expect(latestRuns).toHaveLength(FILE_COUNT * 2); + }); + + it("delivers every output SILENTLY in place β€” workspace never grows past 61", async () => { + await runUntilSettled(FILE_COUNT * 2); + + // 122 deliveries, all silent (background), none via the disruptive path. + expect(mocks.consumeSilentCalls).toBe(FILE_COUNT * 2); + expect(mocks.consumeNonSilentCalls).toBe(0); + // Never added as brand-new files either. + expect(mocks.addFilesCalls).toBe(0); + // In-place versioning: each file replaced twice, count unchanged. + expect(mocks.workspace).toHaveLength(FILE_COUNT); + }); + + it("does NOT re-open files that were closed while their runs were in flight", async () => { + renderHook(() => Harness()); + // Close everything immediately β€” as if the user cleared the workbench mid-run. + // The inputs still persist in storage, so getStirlingFileStub resolves them. + mocks.getStirlingFileStub.mockResolvedValue({ + id: "storage", + versionNumber: 1, + }); + act(() => { + mocks.workspace = []; + }); + + await act(async () => { + await vi.waitFor( + () => { + const imported = latestRuns.filter((r) => r.imported).length; + expect(imported).toBe(FILE_COUNT * 2); + }, + { timeout: 8000, interval: 20 }, + ); + }); + + // Still fully processed (chain intact), but delivered to STORAGE, never + // re-added to the workbench β€” the workspace stays empty. + expect(latestRuns).toHaveLength(FILE_COUNT * 2); + expect(mocks.workspace).toHaveLength(0); + expect(mocks.consumeSilentCalls).toBe(0); + expect(mocks.persistCalls).toBeGreaterThan(0); + }); +}); diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx new file mode 100644 index 0000000000..638c821669 --- /dev/null +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx @@ -0,0 +1,120 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { renderHook, act } from "@testing-library/react"; + +// Two active upload policies, so the auto-run should CHAIN them: fire the first on +// the upload, then the second on the first's output. Stub the contexts + network so +// we can drive the dispatch against the REAL run store. +vi.mock("@app/constants/featureFlags", () => ({ POLICIES_ENABLED: true })); +const fileStubs: { id: string; name: string; derivedFromTool?: boolean }[] = []; +vi.mock("@app/contexts/FileContext", () => ({ + useAllFiles: () => ({ fileStubs }), + useFileManagement: () => ({ addFiles: vi.fn() }), + useFileContext: () => ({ consumeFiles: vi.fn() }), +})); +vi.mock("@app/hooks/usePolicies", () => ({ + usePolicies: () => ({ + policies: { + security: { + configured: true, + status: "active", + backendId: "backend-sec", + runOn: "upload", + order: 0, + }, + classification: { + configured: true, + status: "active", + backendId: "backend-cls", + runOn: "upload", + order: 1, + }, + }, + }), +})); +vi.mock("@app/services/policyApi", () => ({ + runStoredPolicy: vi.fn(), + getPolicyRun: vi.fn(), + downloadPolicyOutput: vi.fn(), + resolvePolicyRunTarget: () => "saas", +})); +vi.mock("@app/services/fileStorage", () => ({ + fileStorage: { getStirlingFile: vi.fn(), getStirlingFileStub: vi.fn() }, +})); +vi.mock("@app/contexts/IndexedDBContext", () => ({ + useIndexedDB: () => ({ bumpRevision: vi.fn() }), +})); + +import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun"; +import { + recordRunStart, + updateRun, + resetPolicyRuns, +} from "@app/components/policies/policyRunStore"; +import { runStoredPolicy } from "@app/services/policyApi"; +import { fileStorage } from "@app/services/fileStorage"; + +const runStored = vi.mocked(runStoredPolicy); +const getFile = vi.mocked(fileStorage.getStirlingFile); + +/** Reset the shared file list between tests without swapping the array identity. */ +function setFileStubs(next: typeof fileStubs) { + fileStubs.length = 0; + fileStubs.push(...next); +} + +beforeEach(() => { + vi.useFakeTimers(); + localStorage.clear(); + resetPolicyRuns(); + setFileStubs([]); + runStored.mockReset(); + getFile.mockReset(); + getFile.mockResolvedValue({ size: 100 } as never); +}); +afterEach(() => vi.useRealTimers()); + +describe("auto-run ordered chaining", () => { + it("dispatches only the FIRST ordered policy on upload, not the whole set", async () => { + setFileStubs([{ id: "file-1", name: "doc.pdf" }]); + runStored.mockResolvedValue("run-sec"); + + renderHook(() => usePolicyAutoRun()); + await act(async () => { + await vi.advanceTimersByTimeAsync(1); + }); + + // The first policy (order 0) runs on the upload; the second waits for the chain. + expect(runStored).toHaveBeenCalledTimes(1); + expect(runStored).toHaveBeenCalledWith("backend-sec", [{ size: 100 }]); + }); + + it("chains the next policy onto a completed run's output", async () => { + // A first-policy run that has completed and imported its output as file-1-v2. + recordRunStart({ + runId: "run-sec", + categoryId: "security", + fileId: "file-1", + fileName: "doc.pdf", + fileSize: 100, + target: "saas", + status: "PENDING", + outputs: [], + error: null, + startedAt: 0, + }); + updateRun("run-sec", { + status: "COMPLETED", + imported: true, + outputFileIds: ["file-1-v2"], + }); + runStored.mockResolvedValue("run-cls"); + + renderHook(() => usePolicyAutoRun()); + await act(async () => { + await vi.advanceTimersByTimeAsync(1); + }); + + // The next policy (order 1) fires on the first policy's output, not the original. + expect(runStored).toHaveBeenCalledWith("backend-cls", [{ size: 100 }]); + }); +}); diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.import.test.tsx b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.import.test.tsx index 249b56ea33..b6e91186fd 100644 --- a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.import.test.tsx +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.import.test.tsx @@ -15,6 +15,7 @@ const mocks = vi.hoisted(() => ({ persistVersionedOutputs: vi.fn(), getStirlingFile: vi.fn(), getStirlingFileStub: vi.fn(), + updateFileMetadata: vi.fn(), downloadPolicyOutput: vi.fn(), listPolicyRuns: vi.fn(), createStirlingFilesAndStubs: vi.fn(), @@ -23,7 +24,10 @@ const mocks = vi.hoisted(() => ({ vi.mock("@app/constants/featureFlags", () => ({ POLICIES_ENABLED: true })); vi.mock("@app/contexts/FileContext", () => ({ useAllFiles: () => ({ fileStubs: mocks.fileStubs }), - useFileManagement: () => ({ addFiles: mocks.addFiles }), + useFileManagement: () => ({ + addFiles: mocks.addFiles, + updateStirlingFileStub: vi.fn(), + }), useFileContext: () => ({ consumeFiles: mocks.consumeFiles }), })); vi.mock("@app/contexts/IndexedDBContext", () => ({ @@ -54,6 +58,7 @@ vi.mock("@app/services/fileStorage", () => ({ fileStorage: { getStirlingFile: mocks.getStirlingFile, getStirlingFileStub: mocks.getStirlingFileStub, + updateFileMetadata: mocks.updateFileMetadata, persistVersionedOutputs: mocks.persistVersionedOutputs, }, })); @@ -100,6 +105,7 @@ beforeEach(() => { mocks.listPolicyRuns.mockResolvedValue([]); mocks.getStirlingFileStub.mockResolvedValue(null); mocks.persistVersionedOutputs.mockResolvedValue(undefined); + mocks.updateFileMetadata.mockResolvedValue(true); mocks.consumeFiles.mockResolvedValue(undefined); mocks.addFiles.mockResolvedValue([{ fileId: "out-1" }]); mocks.downloadPolicyOutput.mockResolvedValue( @@ -133,17 +139,23 @@ describe("auto-run import: new-version output delivery", () => { expect(mocks.addFiles).not.toHaveBeenCalled(); }); - it("versions the input in the workspace when it's open (consumeFiles, not a storage write)", async () => { + it("versions the input in place SILENTLY when it's open (consumeFiles with silent, + sidebar bump)", async () => { mocks.fileStubs = [{ id: "file-1" }]; recordCompletedRun(); await runImport(); + // Background enforcement must not disturb the workbench: the silent consume + // replaces the file in place without auto-selecting / reordering / opening it. expect(mocks.consumeFiles).toHaveBeenCalledWith( ["file-1"], expect.any(Array), expect.any(Array), + { silent: true }, ); + // No redundant bump in the in-workspace path β€” the silent consume's own state + // update drives the sidebar refresh (avoids an O(n) IDB re-read per delivery). + expect(mocks.bumpRevision).not.toHaveBeenCalled(); expect(mocks.persistVersionedOutputs).not.toHaveBeenCalled(); expect(mocks.addFiles).not.toHaveBeenCalled(); }); @@ -166,9 +178,12 @@ describe("auto-run import: new-version output delivery", () => { expect(mocks.consumeFiles).not.toHaveBeenCalled(); }); - it("adopts a server-only run, dating it from the server's createdAt (not now)", async () => { - // A run the client never recorded (true orphan): reconcile adopts it from the server. With no - // local input link it delivers as a new file, and its age comes from the server, not Date.now(). + it("adopts a server-only run for visibility, without delivering its outputs", async () => { + // A run the client never recorded (true orphan): reconcile adopts it from the server for the + // activity feed, dated from the server's createdAt (not Date.now()). It is adopted already + // `imported` β€” with no local input link a delivery would add its output as a NEW workspace + // file, and since cap-evicted runs are re-adopted on every refresh, that meant phantom + // duplicates opening onto the workbench after each reload. mocks.listPolicyRuns.mockResolvedValue([ { runId: "srv-1", @@ -188,10 +203,8 @@ describe("auto-run import: new-version output delivery", () => { }); expect(getRun("srv-1")?.startedAt).toBe(1000); - expect(mocks.addFiles).toHaveBeenCalledWith( - expect.any(Array), - expect.objectContaining({ derivedFromTool: true }), - ); + expect(mocks.addFiles).not.toHaveBeenCalled(); + expect(mocks.downloadPolicyOutput).not.toHaveBeenCalled(); expect(mocks.persistVersionedOutputs).not.toHaveBeenCalled(); }); }); diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts index 9ecdf62525..e17947f59e 100644 --- a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts @@ -1,9 +1,13 @@ /** - * Auto-run controller: every enabled policy enforces on every uploaded - * file. Watches the session's files and, for each (active policy Γ— not-yet-run - * file), fires a real backend run (`POST /api/v1/policies/{id}/run`) and polls it - * to completion, recording progress in {@link policyRunStore} for the activity - * feed. + * Auto-run controller: every enabled policy enforces on every uploaded file. + * Watches the session's files and fires a real backend run + * (`POST /api/v1/policies/{id}/run`) per file, polling it to completion and + * recording progress in {@link policyRunStore} for the activity feed. + * + * When several policies enforce on the same trigger they run as an ordered chain: + * the first fires on the upload, and each subsequent policy fires on the previous + * one's output once it lands β€” so their effects accumulate in the admin-defined + * order rather than racing to fork the same version. * * Headless β€” call it from {@link PolicyAutoRunController}, which is mounted once * wherever the editor is open so enforcement happens regardless of whether the @@ -11,7 +15,7 @@ * in the run store), so re-renders and remounts don't re-fire. */ -import { useCallback, useEffect, useRef } from "react"; +import { useCallback, useEffect, useMemo, useRef } from "react"; import { useAllFiles, useFileManagement, @@ -35,6 +39,7 @@ import type { import { dispatchPaygLimitReached } from "@app/services/usageLimitBridge"; import type { FileId } from "@app/types/file"; import { createStirlingFilesAndStubs } from "@app/services/fileStubHelpers"; +import { readClassificationLabelsFromFile } from "@app/services/fileClassification"; import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext"; import type { PoliciesByCategory } from "@app/types/policies"; import { usePolicies } from "@app/hooks/usePolicies"; @@ -120,11 +125,18 @@ const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export function usePolicyAutoRun(): void { const { fileStubs } = useAllFiles(); - const { addFiles } = useFileManagement(); + const { addFiles, updateStirlingFileStub } = useFileManagement(); const { consumeFiles } = useFileContext(); const { bumpRevision } = useIndexedDB(); const { policies } = usePolicies(); const runs = usePolicyRuns(); + // Live view of the workspace files, read inside the import effect WITHOUT making + // it a dependency. The silent consume that delivers an output mutates fileStubs, + // so if the import effect depended on fileStubs it would re-fire on its own + // delivery β€” an infinite import cascade (and a bumpRevision storm that trips + // React's max-update-depth). The effect only needs to fire when `runs` changes. + const fileStubsRef = useRef(fileStubs); + fileStubsRef.current = fileStubs; // Keys (run ids / dispatch keys) currently in flight, so the effects never // double-fire across re-renders while their first async step is pending. const polling = useRef>(new Set()); @@ -141,6 +153,33 @@ export function usePolicyAutoRun(): void { // run so a folder-watch burst opens the modal once, not once per file. const firedLimitModal = useRef>(new Set()); + // Active upload policies in execution order. When several enforce on upload they + // run as a chain β€” the first fires on the upload, each subsequent one on the + // previous policy's output β€” so their effects accumulate in a defined order + // instead of racing to fork the same version. Mirrors the dispatch filter + // (incl. the editor-source gate) so the chain honours the same eligibility. + const orderedUploadCategories = useMemo( + () => + Object.entries(policies) + .filter( + ([, s]) => + s.configured && + s.status === "active" && + s.backendId && + (!s.sources || + s.sources.length === 0 || + s.sources.includes("editor")) && + (s.runOn ?? "upload") === "upload", + ) + .sort(([, a], [, b]) => (a.order ?? 0) - (b.order ?? 0)) + .map(([id]) => id), + [policies], + ); + + // Runs whose chain-continuation we've already handled this session, so the next + // policy is dispatched exactly once per completed run. + const chained = useRef>(new Set()); + // Latest policies, read from inside the stable retry callback (which has no deps). const policiesRef = useRef(policies); policiesRef.current = policies; @@ -204,50 +243,75 @@ export function usePolicyAutoRun(): void { [scheduleQueueRetry], ); - // Dispatch: for each active policy Γ— each session file not yet run, fire a run. + // Dispatch: fire only the FIRST upload policy on each not-yet-run file. The rest + // of the chain is dispatched by the chaining effect below, each on the previous + // policy's output, so the policies apply cumulatively in order. useEffect(() => { if (!POLICIES_ENABLED) return; - const active = Object.entries(policies).filter( - ([, s]) => - s.configured && - s.status === "active" && - s.backendId && - // Only enforce in the editor when the policy includes "editor" as a source. - // runOn is an editor-specific parameter: "upload" fires here, "export" fires - // at export time via policyExport. Non-editor sources have their own triggers. - (!s.sources || - s.sources.length === 0 || - s.sources.includes("editor")) && - (s.runOn ?? "upload") === "upload", - ); - for (const [categoryId, s] of active) { - for (const stub of fileStubs) { - // Input-mode policies enforce only on files that actually entered the - // system as an upload β€” not on files a tool/automation produced in-app - // (versioned edits or independent artifacts like convert/split/merge). - // Those are enforced only by export-mode policies, at export time. - if (stub.derivedFromTool) continue; - const key = dispatchKey(categoryId, stub.id); - // Skip if already run (persisted) or a dispatch is in flight β€” the - // in-memory guard prevents double-firing during the async wait. - if (isDispatched(categoryId, stub.id) || dispatching.current.has(key)) { - continue; - } - dispatching.current.add(key); + const firstCategory = orderedUploadCategories[0]; + if (!firstCategory) return; + const backendId = policies[firstCategory]?.backendId; + if (!backendId) return; + for (const stub of fileStubs) { + // Input-mode policies enforce only on files that actually entered the + // system as an upload β€” not on files a tool/automation produced in-app + // (versioned edits or independent artifacts like convert/split/merge). + // Those are enforced only by export-mode policies, at export time. + if (stub.derivedFromTool) continue; + const key = dispatchKey(firstCategory, stub.id); + // Skip if already run (persisted) or a dispatch is in flight β€” the + // in-memory guard prevents double-firing during the async wait. + if ( + isDispatched(firstCategory, stub.id) || + dispatching.current.has(key) + ) { + continue; + } + dispatching.current.add(key); + void runPolicyOnFile(firstCategory, backendId, stub.id, stub.name) + .catch(() => { + // runPolicyOnFile handles its own failures; this is just a backstop + // so an unexpected rejection never becomes an unhandled rejection. + }) + .finally(() => dispatching.current.delete(key)); + } + }, [fileStubs, policies, orderedUploadCategories]); + + // Chain: once a run has completed AND its output landed in the workspace, fire the + // next upload policy on that output. Only chains on success (a failed run has no + // output), and only once per run. isDispatched guards re-dispatch across reloads. + useEffect(() => { + if (!POLICIES_ENABLED) return; + for (const run of runs) { + if (run.status !== "COMPLETED" || !run.imported) continue; + if (chained.current.has(run.runId)) continue; + const nextCategory = nextUploadCategory( + orderedUploadCategories, + run.categoryId, + ); + const outputIds = run.outputFileIds ?? []; + if (!nextCategory || outputIds.length === 0) { + // End of the chain (or nothing to chain onto): don't revisit this run. + chained.current.add(run.runId); + continue; + } + const backendId = policies[nextCategory]?.backendId; + // Next policy not ready yet (still reconciling) β€” retry when policies change. + if (!backendId) continue; + chained.current.add(run.runId); + // Chain onto EVERY output, not just the first β€” a run that produced multiple files (split, + // ZIP-unpacked) must apply the next policy to all of them, or outputs 2..N silently skip it. + for (const outputId of outputIds) { + if (isDispatched(nextCategory, outputId as FileId)) continue; void runPolicyOnFile( - categoryId, - s.backendId as string, - stub.id, - stub.name, - ) - .catch(() => { - // runPolicyOnFile handles its own failures; this is just a backstop - // so an unexpected rejection never becomes an unhandled rejection. - }) - .finally(() => dispatching.current.delete(key)); + nextCategory, + backendId, + outputId as FileId, + run.fileName, + ).catch(() => {}); } } - }, [fileStubs, policies]); + }, [runs, policies, orderedUploadCategories]); // Poll each in-flight run to a terminal state. useEffect(() => { @@ -280,18 +344,32 @@ export function usePolicyAutoRun(): void { const outputMode = policies[run.categoryId]?.outputMode ?? "new_version"; const outputName = policies[run.categoryId]?.outputName ?? ""; const outputNamePosition = policies[run.categoryId]?.outputNamePosition; - const parentStub = fileStubs.find((s) => (s.id as string) === run.fileId); + const parentStub = fileStubsRef.current.find( + (s) => (s.id as string) === run.fileId, + ); void importOutputs(run, { addFiles, consumeFiles, + updateStirlingFileStub, bumpRevision, outputMode, outputName, outputNamePosition, parentStub, + firstUploadCategory: orderedUploadCategories[0], }).finally(() => importing.current.delete(run.runId)); } - }, [runs, addFiles, consumeFiles, policies, fileStubs]); + // NB: fileStubs is intentionally NOT a dependency β€” it's read via a ref so a + // delivery's own workspace mutation can't re-trigger this effect (see the ref + // declaration above). The effect fires on run completions, which is all it needs. + }, [ + runs, + addFiles, + consumeFiles, + updateStirlingFileStub, + policies, + orderedUploadCategories, + ]); // Reconcile against the backend on load. The server owns runs (durable, user-scoped), // so a run started before this client recorded it, or before a refresh/crash, is @@ -315,7 +393,13 @@ interface ImportContext { inputFileIds: FileId[], outputs: StirlingFile[], stubs: StirlingFileStub[], + options?: { silent?: boolean }, ) => Promise; + /** Patch a workspace stub in place (used to stamp a new-file output's category). */ + updateStirlingFileStub: ( + fileId: FileId, + updates: Partial, + ) => void; /** Bump the IndexedDB revision so the file views re-read after a storage-only version write. */ bumpRevision: () => void; /** "new_file" adds the output as a separate file; "new_version" versions the input. */ @@ -327,13 +411,18 @@ interface ImportContext { outputNamePosition?: "prefix" | "suffix" | "auto-number"; /** The input file's stub β€” required to version it; absent if it's been removed. */ parentStub: StirlingFileStub | undefined; + /** The first upload policy in the chain β€” the only one the dispatch effect ever + * fires. Every policy output is marked dispatched for it so a downstream policy's + * output is never mistaken for a fresh upload and re-enforced (an endless loop). */ + firstUploadCategory: string | undefined; } /** * Pull the caller's server-side runs and fold them into the local store. For a run we already * track, patch its status/outputs (preserving local import progress + attribution); for one we - * don't, adopt it so the poll/import effects pick it up. Server-excluded ad-hoc runs and runs we - * can't map to a configured category are skipped. + * don't, adopt it for feed visibility (polled if still live, but never auto-imported β€” see the + * `imported` note below). Server-excluded ad-hoc runs and runs we can't map to a configured + * category are skipped. */ function applyOutputName( inputFileName: string, @@ -343,10 +432,21 @@ function applyOutputName( const dot = inputFileName.lastIndexOf("."); const base = dot > 0 ? inputFileName.slice(0, dot) : inputFileName; const ext = dot > 0 ? inputFileName.slice(dot) : ""; - if (position === "suffix") return `${base}_${outputName}${ext}`; - if (position === "prefix") return `${outputName}_${base}${ext}`; - // auto-number requires dedup state not available here β€” fall back to suffix. - return `${base}_${outputName}${ext}`; + // auto-number needs dedup state not available here, so it falls back to suffix. + return position === "prefix" + ? `${outputName}_${base}${ext}` + : `${base}_${outputName}${ext}`; +} + +/** The next upload policy after {@code categoryId} in the chain, or undefined if + * it's last or no longer in the ordered set (e.g. paused since it ran). */ +function nextUploadCategory( + orderedUploadCategories: string[], + categoryId: string, +): string | undefined { + const index = orderedUploadCategories.indexOf(categoryId); + if (index < 0) return undefined; + return orderedUploadCategories[index + 1]; } async function reconcileServerRuns( @@ -371,10 +471,8 @@ async function reconcileServerRuns( addReconciledRun({ runId: view.runId, categoryId, - // No local input link: a run rediscovered purely from the server (never recorded by this - // client) can't be tied back to a workspace/storage file, so its output is delivered as a - // new file rather than a version, and it isn't retried. The recorded-run path (real fileId) - // covers the common refresh case; this only bites true orphans (storage wipe / other device). + // No local input link: a run rediscovered purely from the server was never recorded by + // this client, so it can't be tied back to a workspace/storage file (and isn't retried). fileId: "", fileName: view.outputs[0]?.fileName ?? "", fileSize: 0, @@ -384,6 +482,8 @@ async function reconcileServerRuns( status: view.status, outputs: view.outputs, error: view.error, + // Adopted for feed visibility ONLY, never delivery: else a completed run evicted from the capped store gets re-adopted every refresh and re-delivered as a new file (no fileId β†’ no parent), opening phantom duplicates forever. Client-recorded runs (real fileId) still deliver. + imported: true, // Use the server's creation time, not now, so a rediscovered run shows its real age. startedAt: view.createdAt, }); @@ -473,6 +573,19 @@ async function importOutputs( return; // transient/mixed: retry the lot later; permanent: already failed. } + // Mark a delivered output as already-handled so the auto-run never re-enforces + // a policy on its own output. Covers the producing policy AND the first upload + // policy (the only one the dispatch effect fires) β€” without the latter, a + // downstream policy's output looks like a fresh upload and the first policy + // re-runs on it, versioning/duplicating endlessly. Forward chaining is + // unaffected: it only ever fires categories AFTER the producer, never the first. + const markHandled = (id: string) => { + markDispatched(run.categoryId, id); + if (ctx.firstUploadCategory && ctx.firstUploadCategory !== run.categoryId) { + markDispatched(ctx.firstUploadCategory, id); + } + }; + // Deliver, then mark exactly those imported. If delivery throws we don't mark // them, so they retry (without having been added). const files = fetched.map((f) => f.file); @@ -489,6 +602,21 @@ async function importOutputs( (await fileStorage.getStirlingFileStub(run.fileId as FileId)) ?? undefined) : undefined; + + // Resolve each output's classification labels and put them ON the stub, so + // they ride through consume/persist to BOTH the workspace and storage β€” and + // every later version inherits them (createChildStub + the CONSUME_FILES + // reducer). This keeps files in their label groups instead of flashing into + // "Other" and waiting on a PDF re-read when a 2nd policy or a tool runs. + // Prefer the input's carried-forward labels (cheap) and only read the + // freshly-labelled file when there's nothing to inherit (the classification + // origin) β€” so a 60-file batch doesn't re-read every downstream output. + const parentLabels = parentStub?.classificationLabels; + const resolveLabels = async (file: File) => + (parentLabels && parentLabels.length > 0 ? parentLabels : undefined) ?? + (await readClassificationLabelsFromFile(file)) ?? + undefined; + if (parentStub) { // Replace the input file with a versioned child (preserves its history). // The version records "automate" as its origin tool β€” a policy is a @@ -498,25 +626,54 @@ async function importOutputs( parentStub, "automate", ); - // derivedFromTool is the durable cross-session guard; markDispatched is the - // belt-and-suspenders session guard. Both are needed: dispatched lives only - // in localStorage (wiped on clear / absent on a different device), while - // derivedFromTool is stamped on the stub itself. - for (const s of stubs) { - s.derivedFromTool = true; - markDispatched(run.categoryId, s.id); - } - deliveredIds = stubs.map((s) => s.id as string); + // Transitive provenance for the PERSISTED record, mirroring what the + // CONSUME_FILES reducer computes for workspace state: the output derives + // from its input plus everything that input derived from. Without this the + // stored lineage misses intermediate hops, and a closed file's policy + // badges can't resolve past the most recent run in a 3+-policy chain. + const lineage = Array.from( + new Set([run.fileId as FileId, ...(parentStub.sourceFileIds ?? [])]), + ); + // Stamp each output stub with: the resolved labels (createChildStub already + // inherited the parent's; this also captures the classification origin, + // where the parent had none but the labelled file does), the transitive + // lineage, and derivedFromTool β€” the durable cross-session guard that stops + // the auto-run ever re-enforcing a policy on its own output (survives a + // localStorage wipe / a different device, unlike the dispatched markers). + const categorized = await Promise.all( + stubs.map(async (s, i) => { + const labels = await resolveLabels(files[i]); + return { + ...s, + sourceFileIds: lineage, + derivedFromTool: true, + ...(labels ? { classificationLabels: labels } : {}), + }; + }), + ); + // Mark the outputs handled BEFORE adding them (belt-and-suspenders session + // guard on top of derivedFromTool) so the auto-run never enforces the policy + // on its own output β€” that would version endlessly in a loop. + for (const s of categorized) markHandled(s.id as string); + deliveredIds = categorized.map((s) => s.id as string); if (ctx.parentStub) { - // Input is in the active workspace: version it there (workspace + storage). - await ctx.consumeFiles([run.fileId as FileId], stirlingFiles, stubs); + // Input is in the active workspace: version it in place, silently β€” the + // output replaces the input in the same slot without being auto-selected, + // reordered to the top, or opened in the viewer. The category rides on the + // stub, so it lands in the right group instantly (no re-read, no flicker). + await ctx.consumeFiles( + [run.fileId as FileId], + stirlingFiles, + categorized, + { silent: true }, + ); } else { // Input is only in storage (run recovered after a reload): version it at the // storage layer, then refresh the file views. await fileStorage.persistVersionedOutputs( [run.fileId as FileId], stirlingFiles, - stubs, + categorized, ); ctx.bumpRevision(); } @@ -528,8 +685,28 @@ async function importOutputs( derivedFromTool: true, }); // Belt-and-suspenders session guard on top of derivedFromTool. - for (const f of added) markDispatched(run.categoryId, f.fileId); + for (const f of added) markHandled(f.fileId as string); deliveredIds = added.map((f) => f.fileId as string); + // Mark each new-file output as tool-derived (the versioned path gets this from the + // CONSUME_FILES reducer; the addFiles path doesn't). This is the real loop guard: the dispatch + // effect skips `derivedFromTool` files, so a policy output is never re-enforced as a fresh + // upload regardless of how upload policies are later reordered β€” unlike per-(category,file) + // markers keyed to whichever policy is currently first. Also stamp labels so it lands in the + // right sidebar group immediately (a new file has no parent to inherit from). + let mutated = false; + await Promise.all( + added.map(async (f, i) => { + const labels = await resolveLabels(files[i]); + const updates = { + derivedFromTool: true, + ...(labels ? { classificationLabels: labels } : {}), + }; + ctx.updateStirlingFileStub(f.fileId, updates); + const ok = await fileStorage.updateFileMetadata(f.fileId, updates); + if (ok) mutated = true; + }), + ); + if (mutated) ctx.bumpRevision(); } const importedFileIds = [...done, ...fetched.map((f) => f.fileId)]; const imported = run.outputs.every((out) => diff --git a/frontend/editor/src/proprietary/components/shared/PolicyEnforcingOverlay.tsx b/frontend/editor/src/proprietary/components/shared/PolicyEnforcingOverlay.tsx index 973410aef3..92cc4a556f 100644 --- a/frontend/editor/src/proprietary/components/shared/PolicyEnforcingOverlay.tsx +++ b/frontend/editor/src/proprietary/components/shared/PolicyEnforcingOverlay.tsx @@ -20,6 +20,9 @@ interface PolicyEnforcingOverlayProps { zIndex?: number; /** When provided, an Γ— button is shown and called on click. */ onDismiss?: () => void; + /** CSS colour var of the enforcing policy's accent (e.g. `var(--color-orange)`), + * so the icon/spinner match that policy's badge instead of a fixed blue. */ + accentVar?: string; } /** @@ -31,6 +34,7 @@ export function PolicyEnforcingOverlay({ progress, zIndex = 200, onDismiss, + accentVar, }: PolicyEnforcingOverlayProps) { const { t } = useTranslation(); if (!enforcing) return null; @@ -66,7 +70,23 @@ export function PolicyEnforcingOverlay({ )}
- + @@ -80,9 +100,10 @@ export function PolicyEnforcingOverlay({ value={progress} striped animated + color={accentVar} /> ) : ( - + )}
diff --git a/frontend/editor/src/proprietary/components/viewer/PolicyEnforcementOverlay.tsx b/frontend/editor/src/proprietary/components/viewer/PolicyEnforcementOverlay.tsx index 53fb1a4733..75fa9cb952 100644 --- a/frontend/editor/src/proprietary/components/viewer/PolicyEnforcementOverlay.tsx +++ b/frontend/editor/src/proprietary/components/viewer/PolicyEnforcementOverlay.tsx @@ -70,6 +70,7 @@ export function PolicyEnforcementOverlay({ runs }: Props) { zIndex={1100} progress={progress} onDismiss={() => setDismissed(true)} + accentVar={policyAccentVar(inFlight.categoryId)} /> ); } diff --git a/frontend/editor/src/proprietary/data/classificationLabels.json b/frontend/editor/src/proprietary/data/classificationLabels.json new file mode 100644 index 0000000000..d22fecc155 --- /dev/null +++ b/frontend/editor/src/proprietary/data/classificationLabels.json @@ -0,0 +1,1719 @@ +{ + "labels": [ + { + "id": "invoice", + "name": "Invoice", + "icon": "receipt-long" + }, + { + "id": "receipt", + "name": "Receipt", + "icon": "receipt" + }, + { + "id": "credit-note", + "name": "Credit note", + "icon": "currency-exchange" + }, + { + "id": "debit-note", + "name": "Debit note", + "icon": "attach-money" + }, + { + "id": "purchase-order", + "name": "Purchase order", + "icon": "shopping-cart" + }, + { + "id": "order-confirmation", + "name": "Order confirmation", + "icon": "shopping-cart" + }, + { + "id": "quote", + "name": "Quote", + "icon": "request-quote" + }, + { + "id": "estimate", + "name": "Estimate", + "icon": "price-check" + }, + { + "id": "proforma-invoice", + "name": "Proforma invoice", + "icon": "paid" + }, + { + "id": "bank-statement", + "name": "Bank statement", + "icon": "account-balance" + }, + { + "id": "financial-statement", + "name": "Financial statement", + "icon": "account-balance-wallet" + }, + { + "id": "balance-sheet", + "name": "Balance sheet", + "icon": "table-chart" + }, + { + "id": "income-statement", + "name": "Income statement", + "icon": "functions" + }, + { + "id": "cash-flow-statement", + "name": "Cash flow statement", + "icon": "query-stats" + }, + { + "id": "expense-report", + "name": "Expense report", + "icon": "credit-card" + }, + { + "id": "budget", + "name": "Budget", + "icon": "savings" + }, + { + "id": "financial-forecast", + "name": "Financial forecast", + "icon": "insights" + }, + { + "id": "payslip", + "name": "Payslip", + "icon": "payments" + }, + { + "id": "payroll-document", + "name": "Payroll document", + "icon": "wallet" + }, + { + "id": "tax-form", + "name": "Tax form", + "icon": "calculate" + }, + { + "id": "tax-return", + "name": "Tax return", + "icon": "percent" + }, + { + "id": "tax-statement", + "name": "Tax statement", + "icon": "calculate" + }, + { + "id": "remittance-advice", + "name": "Remittance advice", + "icon": "send" + }, + { + "id": "payment-reminder", + "name": "Payment reminder", + "icon": "schedule" + }, + { + "id": "statement-of-account", + "name": "Statement of account", + "icon": "inbox" + }, + { + "id": "dunning-letter", + "name": "Dunning letter", + "icon": "mail" + }, + { + "id": "audit-report", + "name": "Audit report", + "icon": "fact-check" + }, + { + "id": "annual-report", + "name": "Annual report", + "icon": "leaderboard" + }, + { + "id": "quarterly-report", + "name": "Quarterly report", + "icon": "bar-chart" + }, + { + "id": "pricing-sheet", + "name": "Pricing sheet", + "icon": "sell" + }, + { + "id": "price-list", + "name": "Price list", + "icon": "format-list-bulleted" + }, + { + "id": "loan-document", + "name": "Loan document", + "icon": "assured-workload" + }, + { + "id": "mortgage-document", + "name": "Mortgage document", + "icon": "home" + }, + { + "id": "investment-summary", + "name": "Investment summary", + "icon": "trending-up" + }, + { + "id": "donation-receipt", + "name": "Donation receipt", + "icon": "volunteer-activism" + }, + { + "id": "contract", + "name": "Contract", + "icon": "handshake" + }, + { + "id": "nda", + "name": "NDA", + "icon": "lock" + }, + { + "id": "service-agreement", + "name": "Service agreement", + "icon": "handshake" + }, + { + "id": "employment-contract", + "name": "Employment contract", + "icon": "work" + }, + { + "id": "lease-agreement", + "name": "Lease agreement", + "icon": "home-work" + }, + { + "id": "rental-agreement", + "name": "Rental agreement", + "icon": "home-work" + }, + { + "id": "license-agreement", + "name": "License agreement", + "icon": "verified" + }, + { + "id": "purchase-agreement", + "name": "Purchase agreement", + "icon": "shopping-cart" + }, + { + "id": "partnership-agreement", + "name": "Partnership agreement", + "icon": "diversity-3" + }, + { + "id": "loan-agreement", + "name": "Loan agreement", + "icon": "account-balance" + }, + { + "id": "settlement-agreement", + "name": "Settlement agreement", + "icon": "balance" + }, + { + "id": "vendor-agreement", + "name": "Vendor agreement", + "icon": "storefront" + }, + { + "id": "franchise-agreement", + "name": "Franchise agreement", + "icon": "business-center" + }, + { + "id": "non-compete-agreement", + "name": "Non-compete agreement", + "icon": "gavel" + }, + { + "id": "amendment", + "name": "Amendment", + "icon": "edit-document" + }, + { + "id": "addendum", + "name": "Addendum", + "icon": "edit-document" + }, + { + "id": "terms-and-conditions", + "name": "Terms and conditions", + "icon": "gavel" + }, + { + "id": "terms-of-service", + "name": "Terms of service", + "icon": "gavel" + }, + { + "id": "privacy-policy", + "name": "Privacy policy", + "icon": "privacy-tip" + }, + { + "id": "power-of-attorney", + "name": "Power of attorney", + "icon": "gavel" + }, + { + "id": "affidavit", + "name": "Affidavit", + "icon": "gavel" + }, + { + "id": "will", + "name": "Will", + "icon": "history-edu" + }, + { + "id": "trust-document", + "name": "Trust document", + "icon": "shield" + }, + { + "id": "deed", + "name": "Deed", + "icon": "home-work" + }, + { + "id": "court-filing", + "name": "Court filing", + "icon": "balance" + }, + { + "id": "legal-brief", + "name": "Legal brief", + "icon": "balance" + }, + { + "id": "legal-opinion", + "name": "Legal opinion", + "icon": "balance" + }, + { + "id": "legal-notice", + "name": "Legal notice", + "icon": "gavel" + }, + { + "id": "subpoena", + "name": "Subpoena", + "icon": "gavel" + }, + { + "id": "cease-and-desist", + "name": "Cease and desist", + "icon": "gavel" + }, + { + "id": "compliance-document", + "name": "Compliance document", + "icon": "rule" + }, + { + "id": "regulatory-filing", + "name": "Regulatory filing", + "icon": "rule" + }, + { + "id": "consent-form", + "name": "Consent form", + "icon": "fact-check" + }, + { + "id": "waiver", + "name": "Waiver", + "icon": "fact-check" + }, + { + "id": "memorandum-of-understanding", + "name": "Memorandum of understanding", + "icon": "handshake" + }, + { + "id": "letter-of-intent", + "name": "Letter of intent", + "icon": "draft" + }, + { + "id": "articles-of-incorporation", + "name": "Articles of incorporation", + "icon": "business-center" + }, + { + "id": "bylaws", + "name": "Bylaws", + "icon": "rule" + }, + { + "id": "shareholder-agreement", + "name": "Shareholder agreement", + "icon": "handshake" + }, + { + "id": "board-resolution", + "name": "Board resolution", + "icon": "groups" + }, + { + "id": "resume", + "name": "Resume", + "icon": "person" + }, + { + "id": "cv", + "name": "CV", + "icon": "person" + }, + { + "id": "cover-letter", + "name": "Cover letter", + "icon": "mail" + }, + { + "id": "job-description", + "name": "Job description", + "icon": "topic" + }, + { + "id": "job-application", + "name": "Job application", + "icon": "note-add" + }, + { + "id": "offer-letter", + "name": "Offer letter", + "icon": "work" + }, + { + "id": "onboarding-document", + "name": "Onboarding document", + "icon": "badge" + }, + { + "id": "employee-handbook", + "name": "Employee handbook", + "icon": "menu-book" + }, + { + "id": "hr-policy", + "name": "HR policy", + "icon": "policy" + }, + { + "id": "performance-review", + "name": "Performance review", + "icon": "monitoring" + }, + { + "id": "timesheet", + "name": "Timesheet", + "icon": "schedule" + }, + { + "id": "leave-request", + "name": "Leave request", + "icon": "event" + }, + { + "id": "resignation-letter", + "name": "Resignation letter", + "icon": "mail" + }, + { + "id": "termination-letter", + "name": "Termination letter", + "icon": "send" + }, + { + "id": "reference-letter", + "name": "Reference letter", + "icon": "contact-page" + }, + { + "id": "recommendation-letter", + "name": "Recommendation letter", + "icon": "how-to-reg" + }, + { + "id": "training-material", + "name": "Training material", + "icon": "school" + }, + { + "id": "organization-chart", + "name": "Organization chart", + "icon": "supervisor-account" + }, + { + "id": "benefits-summary", + "name": "Benefits summary", + "icon": "health-and-safety" + }, + { + "id": "hr-memo", + "name": "HR memo", + "icon": "sticky-note-2" + }, + { + "id": "letter", + "name": "Letter", + "icon": "mail" + }, + { + "id": "email-thread", + "name": "Email thread", + "icon": "alternate-email" + }, + { + "id": "memo", + "name": "Memo", + "icon": "sticky-note-2" + }, + { + "id": "meeting-minutes", + "name": "Meeting minutes", + "icon": "groups" + }, + { + "id": "meeting-agenda", + "name": "Meeting agenda", + "icon": "checklist" + }, + { + "id": "newsletter", + "name": "Newsletter", + "icon": "newspaper" + }, + { + "id": "announcement", + "name": "Announcement", + "icon": "campaign" + }, + { + "id": "notice", + "name": "Notice", + "icon": "campaign" + }, + { + "id": "public-notice", + "name": "Public notice", + "icon": "public" + }, + { + "id": "press-release", + "name": "Press release", + "icon": "campaign" + }, + { + "id": "complaint-letter", + "name": "Complaint letter", + "icon": "forum" + }, + { + "id": "demand-letter", + "name": "Demand letter", + "icon": "mail" + }, + { + "id": "confirmation-letter", + "name": "Confirmation letter", + "icon": "mail" + }, + { + "id": "report", + "name": "Report", + "icon": "bar-chart" + }, + { + "id": "progress-report", + "name": "Progress report", + "icon": "timeline" + }, + { + "id": "status-report", + "name": "Status report", + "icon": "monitoring" + }, + { + "id": "incident-report", + "name": "Incident report", + "icon": "emergency" + }, + { + "id": "inspection-report", + "name": "Inspection report", + "icon": "fact-check" + }, + { + "id": "survey-results", + "name": "Survey results", + "icon": "pie-chart" + }, + { + "id": "market-research", + "name": "Market research", + "icon": "insights" + }, + { + "id": "case-study", + "name": "Case study", + "icon": "article" + }, + { + "id": "white-paper", + "name": "White paper", + "icon": "library-books" + }, + { + "id": "research-paper", + "name": "Research paper", + "icon": "science" + }, + { + "id": "research-abstract", + "name": "Research abstract", + "icon": "text-snippet" + }, + { + "id": "feasibility-study", + "name": "Feasibility study", + "icon": "architecture" + }, + { + "id": "risk-assessment", + "name": "Risk assessment", + "icon": "security" + }, + { + "id": "analytics-report", + "name": "Analytics report", + "icon": "analytics" + }, + { + "id": "sales-report", + "name": "Sales report", + "icon": "trending-up" + }, + { + "id": "expense-summary", + "name": "Expense summary", + "icon": "pie-chart" + }, + { + "id": "board-report", + "name": "Board report", + "icon": "groups" + }, + { + "id": "sustainability-report", + "name": "Sustainability report", + "icon": "eco" + }, + { + "id": "standard-operating-procedure", + "name": "Standard operating procedure", + "icon": "rule" + }, + { + "id": "work-instruction", + "name": "Work instruction", + "icon": "checklist" + }, + { + "id": "manual", + "name": "Manual", + "icon": "menu-book" + }, + { + "id": "user-guide", + "name": "User guide", + "icon": "auto-stories" + }, + { + "id": "quick-start-guide", + "name": "Quick start guide", + "icon": "menu-book" + }, + { + "id": "checklist", + "name": "Checklist", + "icon": "checklist" + }, + { + "id": "inventory-list", + "name": "Inventory list", + "icon": "inventory-2" + }, + { + "id": "stock-report", + "name": "Stock report", + "icon": "warehouse" + }, + { + "id": "packing-slip", + "name": "Packing slip", + "icon": "package-2" + }, + { + "id": "delivery-note", + "name": "Delivery note", + "icon": "local-shipping" + }, + { + "id": "bill-of-lading", + "name": "Bill of lading", + "icon": "pallet" + }, + { + "id": "waybill", + "name": "Waybill", + "icon": "map" + }, + { + "id": "customs-declaration", + "name": "Customs declaration", + "icon": "flight" + }, + { + "id": "customs-form", + "name": "Customs form", + "icon": "assignment" + }, + { + "id": "freight-document", + "name": "Freight document", + "icon": "forklift" + }, + { + "id": "shipping-confirmation", + "name": "Shipping confirmation", + "icon": "send" + }, + { + "id": "supply-order", + "name": "Supply order", + "icon": "shopping-cart" + }, + { + "id": "work-order", + "name": "Work order", + "icon": "task" + }, + { + "id": "maintenance-log", + "name": "Maintenance log", + "icon": "build" + }, + { + "id": "service-report", + "name": "Service report", + "icon": "engineering" + }, + { + "id": "quality-report", + "name": "Quality report", + "icon": "verified" + }, + { + "id": "safety-data-sheet", + "name": "Safety data sheet", + "icon": "health-and-safety" + }, + { + "id": "safety-procedure", + "name": "Safety procedure", + "icon": "health-and-safety" + }, + { + "id": "warehouse-receipt", + "name": "Warehouse receipt", + "icon": "inventory-2" + }, + { + "id": "return-authorization", + "name": "Return authorization", + "icon": "redeem" + }, + { + "id": "proposal", + "name": "Proposal", + "icon": "slideshow" + }, + { + "id": "business-proposal", + "name": "Business proposal", + "icon": "business-center" + }, + { + "id": "sales-proposal", + "name": "Sales proposal", + "icon": "trending-up" + }, + { + "id": "pitch-deck", + "name": "Pitch deck", + "icon": "leaderboard" + }, + { + "id": "presentation", + "name": "Presentation", + "icon": "slideshow" + }, + { + "id": "brochure", + "name": "Brochure", + "icon": "image" + }, + { + "id": "flyer", + "name": "Flyer", + "icon": "palette" + }, + { + "id": "catalog", + "name": "Catalog", + "icon": "menu-book" + }, + { + "id": "product-sheet", + "name": "Product sheet", + "icon": "description" + }, + { + "id": "marketing-plan", + "name": "Marketing plan", + "icon": "campaign" + }, + { + "id": "campaign-brief", + "name": "Campaign brief", + "icon": "campaign" + }, + { + "id": "media-kit", + "name": "Media kit", + "icon": "photo-camera" + }, + { + "id": "promotional-material", + "name": "Promotional material", + "icon": "celebration" + }, + { + "id": "advertisement", + "name": "Advertisement", + "icon": "campaign" + }, + { + "id": "request-for-proposal", + "name": "Request for proposal", + "icon": "assignment" + }, + { + "id": "request-for-quotation", + "name": "Request for quotation", + "icon": "request-quote" + }, + { + "id": "tender-document", + "name": "Tender document", + "icon": "assignment" + }, + { + "id": "statement-of-work", + "name": "Statement of work", + "icon": "task" + }, + { + "id": "scope-of-work", + "name": "Scope of work", + "icon": "task" + }, + { + "id": "specification", + "name": "Specification", + "icon": "engineering" + }, + { + "id": "technical-specification", + "name": "Technical specification", + "icon": "code" + }, + { + "id": "requirements-document", + "name": "Requirements document", + "icon": "checklist" + }, + { + "id": "design-document", + "name": "Design document", + "icon": "design-services" + }, + { + "id": "architecture-document", + "name": "Architecture document", + "icon": "architecture" + }, + { + "id": "datasheet", + "name": "Datasheet", + "icon": "table-chart" + }, + { + "id": "schematic", + "name": "Schematic", + "icon": "bolt" + }, + { + "id": "blueprint", + "name": "Blueprint", + "icon": "construction" + }, + { + "id": "technical-drawing", + "name": "Technical drawing", + "icon": "engineering" + }, + { + "id": "floor-plan", + "name": "Floor plan", + "icon": "apartment" + }, + { + "id": "patent", + "name": "Patent", + "icon": "copyright" + }, + { + "id": "test-plan", + "name": "Test plan", + "icon": "checklist" + }, + { + "id": "test-report", + "name": "Test report", + "icon": "fact-check" + }, + { + "id": "release-notes", + "name": "Release notes", + "icon": "article" + }, + { + "id": "change-log", + "name": "Change log", + "icon": "edit-document" + }, + { + "id": "api-documentation", + "name": "API documentation", + "icon": "api" + }, + { + "id": "bill-of-materials", + "name": "Bill of materials", + "icon": "inventory-2" + }, + { + "id": "business-plan", + "name": "Business plan", + "icon": "business-center" + }, + { + "id": "project-plan", + "name": "Project plan", + "icon": "task" + }, + { + "id": "project-charter", + "name": "Project charter", + "icon": "task" + }, + { + "id": "roadmap", + "name": "Roadmap", + "icon": "trending-up" + }, + { + "id": "timeline", + "name": "Timeline", + "icon": "timeline" + }, + { + "id": "meeting-notes", + "name": "Meeting notes", + "icon": "sticky-note-2" + }, + { + "id": "action-plan", + "name": "Action plan", + "icon": "checklist" + }, + { + "id": "retrospective", + "name": "Retrospective", + "icon": "psychology" + }, + { + "id": "transcript", + "name": "Transcript", + "icon": "school" + }, + { + "id": "diploma", + "name": "Diploma", + "icon": "history-edu" + }, + { + "id": "certificate", + "name": "Certificate", + "icon": "verified" + }, + { + "id": "certificate-of-completion", + "name": "Certificate of completion", + "icon": "verified" + }, + { + "id": "course-syllabus", + "name": "Course syllabus", + "icon": "menu-book" + }, + { + "id": "lesson-plan", + "name": "Lesson plan", + "icon": "school" + }, + { + "id": "assignment-brief", + "name": "Assignment brief", + "icon": "assignment" + }, + { + "id": "exam-paper", + "name": "Exam paper", + "icon": "assignment" + }, + { + "id": "grade-report", + "name": "Grade report", + "icon": "school" + }, + { + "id": "thesis", + "name": "Thesis", + "icon": "science" + }, + { + "id": "dissertation", + "name": "Dissertation", + "icon": "science" + }, + { + "id": "study-guide", + "name": "Study guide", + "icon": "menu-book" + }, + { + "id": "academic-record", + "name": "Academic record", + "icon": "school" + }, + { + "id": "medical-report", + "name": "Medical report", + "icon": "medical-services" + }, + { + "id": "lab-report", + "name": "Lab report", + "icon": "science" + }, + { + "id": "radiology-report", + "name": "Radiology report", + "icon": "monitor-heart" + }, + { + "id": "pathology-report", + "name": "Pathology report", + "icon": "medical-information" + }, + { + "id": "prescription", + "name": "Prescription", + "icon": "medication" + }, + { + "id": "referral-letter", + "name": "Referral letter", + "icon": "stethoscope" + }, + { + "id": "discharge-summary", + "name": "Discharge summary", + "icon": "medical-services" + }, + { + "id": "immunization-record", + "name": "Immunization record", + "icon": "vaccines" + }, + { + "id": "medical-invoice", + "name": "Medical invoice", + "icon": "receipt-long" + }, + { + "id": "insurance-policy", + "name": "Insurance policy", + "icon": "shield" + }, + { + "id": "insurance-claim", + "name": "Insurance claim", + "icon": "security" + }, + { + "id": "insurance-certificate", + "name": "Insurance certificate", + "icon": "approval" + }, + { + "id": "explanation-of-benefits", + "name": "Explanation of benefits", + "icon": "medical-information" + }, + { + "id": "property-listing", + "name": "Property listing", + "icon": "real-estate-agent" + }, + { + "id": "appraisal-report", + "name": "Appraisal report", + "icon": "real-estate-agent" + }, + { + "id": "home-inspection-report", + "name": "Home inspection report", + "icon": "home" + }, + { + "id": "title-document", + "name": "Title document", + "icon": "home-work" + }, + { + "id": "closing-statement", + "name": "Closing statement", + "icon": "real-estate-agent" + }, + { + "id": "tenancy-agreement", + "name": "Tenancy agreement", + "icon": "home-work" + }, + { + "id": "eviction-notice", + "name": "Eviction notice", + "icon": "gavel" + }, + { + "id": "hoa-document", + "name": "HOA document", + "icon": "home-work" + }, + { + "id": "utility-bill", + "name": "Utility bill", + "icon": "bolt" + }, + { + "id": "permit", + "name": "Permit", + "icon": "approval" + }, + { + "id": "license", + "name": "License", + "icon": "verified" + }, + { + "id": "registration-form", + "name": "Registration form", + "icon": "how-to-reg" + }, + { + "id": "application-form", + "name": "Application form", + "icon": "assignment" + }, + { + "id": "government-notice", + "name": "Government notice", + "icon": "campaign" + }, + { + "id": "grant-application", + "name": "Grant application", + "icon": "assignment" + }, + { + "id": "grant-agreement", + "name": "Grant agreement", + "icon": "handshake" + }, + { + "id": "visa-document", + "name": "Visa document", + "icon": "public" + }, + { + "id": "immigration-document", + "name": "Immigration document", + "icon": "badge" + }, + { + "id": "legal-filing", + "name": "Legal filing", + "icon": "balance" + }, + { + "id": "itinerary", + "name": "Itinerary", + "icon": "map" + }, + { + "id": "travel-itinerary", + "name": "Travel itinerary", + "icon": "flight" + }, + { + "id": "booking-confirmation", + "name": "Booking confirmation", + "icon": "hotel" + }, + { + "id": "reservation", + "name": "Reservation", + "icon": "restaurant" + }, + { + "id": "ticket", + "name": "Ticket", + "icon": "confirmation-number" + }, + { + "id": "event-agenda", + "name": "Event agenda", + "icon": "calendar-month" + }, + { + "id": "event-program", + "name": "Event program", + "icon": "calendar-month" + }, + { + "id": "invitation", + "name": "Invitation", + "icon": "mail" + }, + { + "id": "registration-confirmation", + "name": "Registration confirmation", + "icon": "fact-check" + }, + { + "id": "form", + "name": "Form", + "icon": "assignment" + }, + { + "id": "questionnaire", + "name": "Questionnaire", + "icon": "assignment" + }, + { + "id": "survey-form", + "name": "Survey form", + "icon": "checklist" + }, + { + "id": "feedback-form", + "name": "Feedback form", + "icon": "forum" + }, + { + "id": "intake-form", + "name": "Intake form", + "icon": "assignment" + }, + { + "id": "order-form", + "name": "Order form", + "icon": "shopping-cart" + }, + { + "id": "claim-form", + "name": "Claim form", + "icon": "assignment" + }, + { + "id": "warranty-document", + "name": "Warranty document", + "icon": "assured-workload" + }, + { + "id": "membership-document", + "name": "Membership document", + "icon": "badge" + }, + { + "id": "subscription-confirmation", + "name": "Subscription confirmation", + "icon": "fact-check" + }, + { + "id": "gift-certificate", + "name": "Gift certificate", + "icon": "redeem" + }, + { + "id": "sponsorship-agreement", + "name": "Sponsorship agreement", + "icon": "handshake" + }, + { + "id": "petition", + "name": "Petition", + "icon": "history-edu" + }, + { + "id": "agenda", + "name": "Agenda", + "icon": "checklist" + }, + { + "id": "fact-sheet", + "name": "Fact sheet", + "icon": "summarize" + }, + { + "id": "faq-document", + "name": "FAQ document", + "icon": "chat" + }, + { + "id": "glossary", + "name": "Glossary", + "icon": "menu-book" + }, + { + "id": "index", + "name": "Index", + "icon": "format-list-bulleted" + }, + { + "id": "table-of-contents", + "name": "Table of contents", + "icon": "topic" + } + ], + "families": [ + { + "id": "finance", + "name": "Financial", + "icon": "payments", + "labelIds": [ + "invoice", + "receipt", + "credit-note", + "debit-note", + "purchase-order", + "order-confirmation", + "quote", + "estimate", + "proforma-invoice", + "bank-statement", + "financial-statement", + "balance-sheet", + "income-statement", + "cash-flow-statement", + "expense-report", + "budget", + "financial-forecast", + "payslip", + "payroll-document", + "tax-form", + "tax-return", + "tax-statement", + "remittance-advice", + "payment-reminder", + "statement-of-account", + "dunning-letter", + "audit-report", + "annual-report", + "quarterly-report", + "pricing-sheet", + "price-list", + "loan-document", + "mortgage-document", + "investment-summary", + "donation-receipt" + ] + }, + { + "id": "legal", + "name": "Legal", + "icon": "gavel", + "labelIds": [ + "contract", + "nda", + "service-agreement", + "employment-contract", + "lease-agreement", + "rental-agreement", + "license-agreement", + "purchase-agreement", + "partnership-agreement", + "loan-agreement", + "settlement-agreement", + "vendor-agreement", + "franchise-agreement", + "non-compete-agreement", + "amendment", + "addendum", + "terms-and-conditions", + "terms-of-service", + "privacy-policy", + "power-of-attorney", + "affidavit", + "will", + "trust-document", + "deed", + "court-filing", + "legal-brief", + "legal-opinion", + "legal-notice", + "subpoena", + "cease-and-desist", + "compliance-document", + "regulatory-filing", + "consent-form", + "waiver", + "memorandum-of-understanding", + "letter-of-intent", + "articles-of-incorporation", + "bylaws", + "shareholder-agreement", + "board-resolution" + ] + }, + { + "id": "hr", + "name": "HR", + "icon": "badge", + "labelIds": [ + "resume", + "cv", + "cover-letter", + "job-description", + "job-application", + "offer-letter", + "onboarding-document", + "employee-handbook", + "hr-policy", + "performance-review", + "timesheet", + "leave-request", + "resignation-letter", + "termination-letter", + "reference-letter", + "recommendation-letter", + "training-material", + "organization-chart", + "benefits-summary", + "hr-memo" + ] + }, + { + "id": "correspondence", + "name": "Correspondence", + "icon": "mail", + "labelIds": [ + "letter", + "email-thread", + "memo", + "meeting-minutes", + "meeting-agenda", + "newsletter", + "announcement", + "notice", + "public-notice", + "press-release", + "complaint-letter", + "demand-letter", + "confirmation-letter" + ] + }, + { + "id": "reports", + "name": "Reports", + "icon": "monitoring", + "labelIds": [ + "report", + "progress-report", + "status-report", + "incident-report", + "inspection-report", + "survey-results", + "market-research", + "case-study", + "white-paper", + "research-paper", + "research-abstract", + "feasibility-study", + "risk-assessment", + "analytics-report", + "sales-report", + "expense-summary", + "board-report", + "sustainability-report" + ] + }, + { + "id": "operations", + "name": "Operations", + "icon": "local-shipping", + "labelIds": [ + "standard-operating-procedure", + "work-instruction", + "manual", + "user-guide", + "quick-start-guide", + "checklist", + "inventory-list", + "stock-report", + "packing-slip", + "delivery-note", + "bill-of-lading", + "waybill", + "customs-declaration", + "customs-form", + "freight-document", + "shipping-confirmation", + "supply-order", + "work-order", + "maintenance-log", + "service-report", + "quality-report", + "safety-data-sheet", + "safety-procedure", + "warehouse-receipt", + "return-authorization" + ] + }, + { + "id": "sales", + "name": "Marketing", + "icon": "campaign", + "labelIds": [ + "proposal", + "business-proposal", + "sales-proposal", + "pitch-deck", + "presentation", + "brochure", + "flyer", + "catalog", + "product-sheet", + "marketing-plan", + "campaign-brief", + "media-kit", + "promotional-material", + "advertisement", + "request-for-proposal", + "request-for-quotation", + "tender-document", + "statement-of-work", + "scope-of-work" + ] + }, + { + "id": "engineering", + "name": "Engineering", + "icon": "engineering", + "labelIds": [ + "specification", + "technical-specification", + "requirements-document", + "design-document", + "architecture-document", + "datasheet", + "schematic", + "blueprint", + "technical-drawing", + "floor-plan", + "patent", + "test-plan", + "test-report", + "release-notes", + "change-log", + "api-documentation", + "bill-of-materials" + ] + }, + { + "id": "projects", + "name": "Projects", + "icon": "task", + "labelIds": [ + "business-plan", + "project-plan", + "project-charter", + "roadmap", + "timeline", + "meeting-notes", + "action-plan", + "retrospective" + ] + }, + { + "id": "education", + "name": "Education", + "icon": "school", + "labelIds": [ + "transcript", + "diploma", + "certificate", + "certificate-of-completion", + "course-syllabus", + "lesson-plan", + "assignment-brief", + "exam-paper", + "grade-report", + "thesis", + "dissertation", + "study-guide", + "academic-record" + ] + }, + { + "id": "health", + "name": "Medical", + "icon": "medical-services", + "labelIds": [ + "medical-report", + "lab-report", + "radiology-report", + "pathology-report", + "prescription", + "referral-letter", + "discharge-summary", + "immunization-record", + "medical-invoice", + "insurance-policy", + "insurance-claim", + "insurance-certificate", + "explanation-of-benefits" + ] + }, + { + "id": "property", + "name": "Property", + "icon": "real-estate-agent", + "labelIds": [ + "property-listing", + "appraisal-report", + "home-inspection-report", + "title-document", + "closing-statement", + "tenancy-agreement", + "eviction-notice", + "hoa-document", + "utility-bill" + ] + }, + { + "id": "government", + "name": "Government", + "icon": "account-balance", + "labelIds": [ + "permit", + "license", + "registration-form", + "application-form", + "government-notice", + "grant-application", + "grant-agreement", + "visa-document", + "immigration-document", + "legal-filing" + ] + }, + { + "id": "travel", + "name": "Travel", + "icon": "flight", + "labelIds": [ + "itinerary", + "travel-itinerary", + "booking-confirmation", + "reservation", + "ticket", + "event-agenda", + "event-program", + "invitation", + "registration-confirmation" + ] + }, + { + "id": "forms", + "name": "Forms", + "icon": "assignment", + "labelIds": [ + "form", + "questionnaire", + "survey-form", + "feedback-form", + "intake-form", + "order-form", + "claim-form", + "warranty-document", + "membership-document", + "subscription-confirmation", + "gift-certificate", + "sponsorship-agreement", + "petition", + "agenda", + "fact-sheet", + "faq-document", + "glossary", + "index", + "table-of-contents" + ] + } + ] +} diff --git a/frontend/editor/src/proprietary/data/classificationLabels.ts b/frontend/editor/src/proprietary/data/classificationLabels.ts new file mode 100644 index 0000000000..3695255c4e --- /dev/null +++ b/frontend/editor/src/proprietary/data/classificationLabels.ts @@ -0,0 +1,86 @@ +// Default classification labels. The SOURCE OF TRUTH is the co-located static +// JSON (`classificationLabels.json`), imported here and shaped into typed +// objects β€” edit THAT file, not this one. This is the ONLY copy of the label +// data: it seeds a team's editable set and drives the sidebar's grouping, +// icons, and display names. Neither the backend nor the engine keeps a copy. +// +// NOTE: this is only the built-in default vocabulary. A team's own +// (admin-editable) labels live in the backend store and are what the backend +// sends to the engine per classify request β€” the engine holds no vocabulary of +// its own. Edits to a team's set reach Python on the next run, independent of +// this file. +// +// `labels` is the flat set: each has a stable `id` (slug β€” the value on the wire, +// in storage and keyed on) and a human `name` (display, translatable via +// `classification.labels.`); `icon` is presentational only, the engine never +// sees it. `families` are presentational sidebar roll-ups (referencing labels by +// id) the classifier never sees. + +import labelsData from "@app/data/classificationLabels.json"; + +export interface ClassificationLabel { + /** Stable identity (slug) β€” the value on the wire, stored on the doc, and + * keyed on. Independent of the (translatable) display name. */ + id: string; + /** Human display name; the en-US default for `classification.labels.`. */ + name: string; + /** Material Symbols icon key (see `labelIcons.ts`). */ + icon?: string; +} + +export interface LabelFamily { + /** Stable identity for sidebar prefs β€” never rename once shipped. */ + id: string; + /** Group header text shown in the sidebar and the group picker. */ + name: string; + /** Material Symbols icon key (see `labelIcons.ts`). */ + icon: string; + /** The built-in labels this family rolls up in the sidebar. */ + labels: ClassificationLabel[]; +} + +/** Shape of `classificationLabels.json` β€” the flat label set plus the + * presentational family grouping (which references labels by id). */ +interface LabelsFile { + labels: ClassificationLabel[]; + families: { id: string; name: string; icon: string; labelIds: string[] }[]; +} + +const data = labelsData as LabelsFile; + +/** Flat default label set β€” file order, as the classifier/team-seed sees it. */ +export const DEFAULT_CLASSIFICATION_LABELS: ClassificationLabel[] = data.labels; + +const LABEL_BY_ID = new Map(data.labels.map((label) => [label.id, label])); + +/** + * Built-in families with their labels resolved from the flat set by id. Throws + * at module load if a family references an unknown id, so a bad hand-edit of the + * JSON fails fast rather than silently dropping a label from its group. + */ +export const LABEL_FAMILIES: LabelFamily[] = data.families.map((family) => ({ + id: family.id, + name: family.name, + icon: family.icon, + labels: family.labelIds.map((id) => { + const label = LABEL_BY_ID.get(id); + if (!label) { + throw new Error( + `Classification family "${family.id}" references unknown label id "${id}"`, + ); + } + return label; + }), +})); + +/** + * Stable slug id from a label's canonical (English) name β€” used to derive an id + * for a NEW custom label the user types. Built-in ids are authored in the JSON; + * this must stay in sync with the slug rule used to generate them. + */ +export function labelId(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); +} diff --git a/frontend/editor/src/proprietary/data/labelDisplay.ts b/frontend/editor/src/proprietary/data/labelDisplay.ts new file mode 100644 index 0000000000..008f8786a7 --- /dev/null +++ b/frontend/editor/src/proprietary/data/labelDisplay.ts @@ -0,0 +1,17 @@ +// Proprietary override of the label-display seam: resolves a classification +// label id to its display name, translated via `classification.labels.` +// (the built-in English name is the en-US default). Custom team labels aren't in +// the built-in map, so they fall back to the id. + +import { useTranslation } from "react-i18next"; +import { DEFAULT_CLASSIFICATION_LABELS } from "@app/data/classificationLabels"; + +const NAME_BY_ID = new Map( + DEFAULT_CLASSIFICATION_LABELS.map((label) => [label.id, label.name]), +); + +/** Returns a resolver mapping a label id to its (translated) display name. */ +export function useLabelName(): (id: string) => string { + const { t } = useTranslation(); + return (id) => t(`classification.labels.${id}`, NAME_BY_ID.get(id) ?? id); +} diff --git a/frontend/editor/src/proprietary/data/labelIcons.ts b/frontend/editor/src/proprietary/data/labelIcons.ts new file mode 100644 index 0000000000..699f9ba61c --- /dev/null +++ b/frontend/editor/src/proprietary/data/labelIcons.ts @@ -0,0 +1,195 @@ +/** + * Curated palette of icons a classification label can use, shown in the label + * icon picker and rendered in the file sidebar's label groups. Keys are + * Material Symbols names rendered via {@link LocalIcon}. + * + * IMPORTANT: each entry is written as an `icon: "…"` literal so the icon-bundler + * (`scripts/generate-icons.js`, which regex-scans for `icon: "name"`) picks every + * one up and bundles it β€” otherwise a picked icon would fall back to the CDN and + * render blank offline. After adding/removing entries run `task frontend:prepare:icons`. + * + * Search is intentionally omitted for now (no synonyms to maintain); the palette + * is small enough to eyeball. + */ + +export interface LabelIconOption { + /** Material Symbols key (no `material-symbols:` prefix). */ + icon: string; + /** Short English name β€” the en-US default for the icon's tooltip/aria; the + * picker translates it via `policies.labels.iconName.`. */ + label: string; +} + +/** Fallback icon for a label with none set (and for the "Other" group). */ +export const DEFAULT_LABEL_ICON = "sell"; + +export const LABEL_ICON_OPTIONS: LabelIconOption[] = [ + // Money / finance + { icon: "receipt-long", label: "Receipt" }, + { icon: "request-quote", label: "Quote" }, + { icon: "payments", label: "Payments" }, + { icon: "account-balance", label: "Bank" }, + { icon: "savings", label: "Savings" }, + { icon: "credit-card", label: "Card" }, + { icon: "point-of-sale", label: "Point of sale" }, + { icon: "sell", label: "Label" }, + { icon: "shopping-cart", label: "Cart" }, + { icon: "calculate", label: "Calculate" }, + // Legal / compliance + { icon: "handshake", label: "Agreement" }, + { icon: "gavel", label: "Legal" }, + { icon: "balance", label: "Balance" }, + { icon: "policy", label: "Policy" }, + { icon: "verified", label: "Verified" }, + { icon: "shield", label: "Shield" }, + { icon: "health-and-safety", label: "Safety" }, + { icon: "lock", label: "Lock" }, + { icon: "fact-check", label: "Fact check" }, + { icon: "rule", label: "Rule" }, + // Documents / writing + { icon: "description", label: "Document" }, + { icon: "article", label: "Article" }, + { icon: "assignment", label: "Form" }, + { icon: "checklist", label: "Checklist" }, + { icon: "task", label: "Task" }, + { icon: "summarize", label: "Summary" }, + { icon: "sticky-note-2", label: "Note" }, + { icon: "edit-document", label: "Edit" }, + { icon: "draft", label: "Draft" }, + { icon: "folder", label: "Folder" }, + // People / comms + { icon: "mail", label: "Mail" }, + { icon: "campaign", label: "Campaign" }, + { icon: "contact-page", label: "Contact" }, + { icon: "badge", label: "Badge" }, + { icon: "groups", label: "Group" }, + { icon: "person", label: "Person" }, + { icon: "work", label: "Work" }, + { icon: "business-center", label: "Business" }, + { icon: "event", label: "Event" }, + { icon: "calendar-month", label: "Calendar" }, + // Charts / education / logistics / misc + { icon: "bar-chart", label: "Bar chart" }, + { icon: "monitoring", label: "Monitoring" }, + { icon: "pie-chart", label: "Pie chart" }, + { icon: "trending-up", label: "Trend" }, + { icon: "table-chart", label: "Table" }, + { icon: "school", label: "School" }, + { icon: "menu-book", label: "Book" }, + { icon: "science", label: "Science" }, + { icon: "medical-services", label: "Medical" }, + { icon: "local-shipping", label: "Shipping" }, + { icon: "slideshow", label: "Presentation" }, + { icon: "home-work", label: "Property" }, + { icon: "real-estate-agent", label: "Real estate" }, + { icon: "engineering", label: "Technical" }, + { icon: "inventory-2", label: "Inventory" }, + { icon: "image", label: "Image" }, + // Money / finance (extended) + { icon: "paid", label: "Paid" }, + { icon: "attach-money", label: "Money" }, + { icon: "currency-exchange", label: "Exchange" }, + { icon: "account-balance-wallet", label: "Wallet" }, + { icon: "price-check", label: "Price check" }, + { icon: "wallet", label: "Billfold" }, + { icon: "redeem", label: "Redeem" }, + { icon: "receipt", label: "Receipt (short)" }, + { icon: "percent", label: "Percent" }, + // Legal / security (extended) + { icon: "encrypted", label: "Encrypted" }, + { icon: "privacy-tip", label: "Privacy" }, + { icon: "security", label: "Security" }, + { icon: "assured-workload", label: "Assured" }, + { icon: "copyright", label: "Copyright" }, + { icon: "approval", label: "Approval" }, + { icon: "history-edu", label: "Quill" }, + // Documents (extended) + { icon: "topic", label: "Topic" }, + { icon: "note-add", label: "New note" }, + { icon: "text-snippet", label: "Snippet" }, + { icon: "library-books", label: "Library" }, + { icon: "auto-stories", label: "Open book" }, + { icon: "newspaper", label: "Newspaper" }, + { icon: "folder-open", label: "Open folder" }, + { icon: "format-list-bulleted", label: "List" }, + // People / HR (extended) + { icon: "manage-accounts", label: "Manage accounts" }, + { icon: "supervisor-account", label: "Supervisor" }, + { icon: "how-to-reg", label: "Registered" }, + { icon: "diversity-3", label: "Team" }, + { icon: "psychology", label: "Psychology" }, + { icon: "volunteer-activism", label: "Volunteer" }, + // Health (extended) + { icon: "medical-information", label: "Medical info" }, + { icon: "medication", label: "Medication" }, + { icon: "vaccines", label: "Vaccine" }, + { icon: "monitor-heart", label: "Heart monitor" }, + { icon: "emergency", label: "Emergency" }, + { icon: "stethoscope", label: "Stethoscope" }, + // Property / places (extended) + { icon: "home", label: "Home" }, + { icon: "apartment", label: "Apartment" }, + { icon: "storefront", label: "Storefront" }, + { icon: "location-on", label: "Location" }, + { icon: "map", label: "Map" }, + { icon: "public", label: "Globe" }, + // Logistics (extended) + { icon: "package-2", label: "Package" }, + { icon: "warehouse", label: "Warehouse" }, + { icon: "forklift", label: "Forklift" }, + { icon: "pallet", label: "Pallet" }, + { icon: "flight", label: "Flight" }, + { icon: "luggage", label: "Luggage" }, + // Technology / engineering (extended) + { icon: "code", label: "Code" }, + { icon: "terminal", label: "Terminal" }, + { icon: "database", label: "Database" }, + { icon: "cloud", label: "Cloud" }, + { icon: "api", label: "API" }, + { icon: "bug-report", label: "Bug" }, + { icon: "build", label: "Wrench" }, + { icon: "architecture", label: "Compass" }, + { icon: "design-services", label: "Design" }, + { icon: "precision-manufacturing", label: "Robotics" }, + { icon: "factory", label: "Factory" }, + { icon: "bolt", label: "Bolt" }, + { icon: "construction", label: "Construction" }, + // Charts / time (extended) + { icon: "analytics", label: "Analytics" }, + { icon: "insights", label: "Insights" }, + { icon: "query-stats", label: "Stats" }, + { icon: "leaderboard", label: "Leaderboard" }, + { icon: "functions", label: "Functions" }, + { icon: "schedule", label: "Clock" }, + { icon: "pending-actions", label: "Pending" }, + { icon: "timeline", label: "Timeline" }, + // Communication (extended) + { icon: "chat", label: "Chat" }, + { icon: "forum", label: "Forum" }, + { icon: "call", label: "Call" }, + { icon: "send", label: "Send" }, + { icon: "inbox", label: "Inbox" }, + { icon: "alternate-email", label: "At sign" }, + // Events / travel (extended) + { icon: "airplane-ticket", label: "Plane ticket" }, + { icon: "confirmation-number", label: "Ticket stub" }, + { icon: "hotel", label: "Hotel" }, + { icon: "restaurant", label: "Restaurant" }, + { icon: "celebration", label: "Celebration" }, + // Media / creative (extended) + { icon: "photo-camera", label: "Camera" }, + { icon: "videocam", label: "Video" }, + { icon: "music-note", label: "Music" }, + { icon: "palette", label: "Palette" }, + { icon: "brush", label: "Brush" }, + { icon: "movie", label: "Movie" }, + // Nature / misc (extended) + { icon: "eco", label: "Eco" }, + { icon: "recycling", label: "Recycling" }, + { icon: "agriculture", label: "Agriculture" }, +]; + +/** Set of valid palette keys, for validating a stored/imported icon. */ +export const LABEL_ICON_KEYS: ReadonlySet = new Set( + LABEL_ICON_OPTIONS.map((option) => option.icon), +); diff --git a/frontend/editor/src/proprietary/data/policyDefinitions.tsx b/frontend/editor/src/proprietary/data/policyDefinitions.tsx index ce12ee3380..837287cbdb 100644 --- a/frontend/editor/src/proprietary/data/policyDefinitions.tsx +++ b/frontend/editor/src/proprietary/data/policyDefinitions.tsx @@ -16,6 +16,7 @@ import PublicIcon from "@mui/icons-material/Public"; import CloudIcon from "@mui/icons-material/Cloud"; import EmailOutlinedIcon from "@mui/icons-material/EmailOutlined"; import FolderOpenIcon from "@mui/icons-material/FolderOpen"; +import LabelOutlinedIcon from "@mui/icons-material/LabelOutlined"; import type { PolicyCategory, PolicyConfigDef, @@ -42,6 +43,14 @@ export const POLICY_CATEGORIES: PolicyCategory[] = [ icon: , desc: "Detect PII, encrypt, verify authenticity, control access, and certify documents.", }, + { + id: "classification", + label: "Classification", + icon: , + desc: "Identify each document's type on upload and tag its metadata for filing and search.", + // Needs the AI engine to classify; hidden from the policy list when it's off. + requiresAiEngine: true, + }, { id: "compliance", label: "Compliance", @@ -204,6 +213,16 @@ export const POLICY_CONFIG: Record = { // output naming + retries are set in the wizard. fields: [], }, + classification: { + summary: + "Classifies every uploaded document and writes the result to its metadata.", + rules: ["Classify", "Tag metadata"], + // Single backend step: classify the document via the AI engine and store the + // result in the document's StirlingPDFClassification metadata field. + defaultOperations: [{ operation: "classify", parameters: {} }], + scopeLabel: "All PDFs on this device", + fields: [], + }, compliance: { summary: "Validates documents against regulatory frameworks before they leave the system.", diff --git a/frontend/editor/src/proprietary/hooks/useClassificationLabels.ts b/frontend/editor/src/proprietary/hooks/useClassificationLabels.ts new file mode 100644 index 0000000000..5f034ad7fc --- /dev/null +++ b/frontend/editor/src/proprietary/hooks/useClassificationLabels.ts @@ -0,0 +1,85 @@ +/** + * Loads and persists the team's classification labels + * (`/api/v1/classification/labels`) β€” one server-truth set shared by the whole + * team. When the team has none, this shows the built-in default as a starting + * point for the editor/sidebar; note the classifier itself does NOT use that + * default β€” the backend only classifies against a team's saved set (an unsaved + * team is not classified). Editing is gated to team leaders / admins by the + * backend β€” callers pass `canConfigure` (the policy gate) to keep read-only + * users out of the save path. + * + * `teamLabels` is also what the sidebar/editor display uses β€” the team set, or + * the built-in default when the team has none. + */ + +import { useCallback, useEffect, useState } from "react"; +import { + DEFAULT_CLASSIFICATION_LABELS, + type ClassificationLabel, +} from "@app/data/classificationLabels"; +import { fetchTeamLabels, saveTeamLabels } from "@app/services/labelsBackend"; + +export interface UseClassificationLabels { + /** Server-truth team labels (or the built-in default when the team has none). */ + teamLabels: ClassificationLabel[]; + /** Whether the team has a stored set (vs. the built-in default). */ + isCustom: boolean; + loading: boolean; + saving: boolean; + /** Last save failure, cleared on the next attempt. */ + error: string | null; + /** Persist the team set; resolves once server state is updated. */ + saveTeam: (next: ClassificationLabel[]) => Promise; +} + +export function useClassificationLabels( + enabled: boolean, +): UseClassificationLabels { + const [teamLabels, setTeamLabels] = useState( + DEFAULT_CLASSIFICATION_LABELS, + ); + const [isCustom, setIsCustom] = useState(false); + const [loading, setLoading] = useState(enabled); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!enabled) return; + let cancelled = false; + setLoading(true); + void (async () => { + const team = await fetchTeamLabels().catch(() => null); + if (cancelled) return; + setTeamLabels(team ?? DEFAULT_CLASSIFICATION_LABELS); + setIsCustom(team != null); + setLoading(false); + })(); + return () => { + cancelled = true; + }; + }, [enabled]); + + const saveTeam = useCallback(async (next: ClassificationLabel[]) => { + setSaving(true); + setError(null); + try { + const saved = await saveTeamLabels(next); + setTeamLabels(saved); + setIsCustom(true); + } catch (e) { + setError(e instanceof Error ? e.message : "Couldn't save the labels."); + throw e; + } finally { + setSaving(false); + } + }, []); + + return { + teamLabels, + isCustom, + loading, + saving, + error, + saveTeam, + }; +} diff --git a/frontend/editor/src/proprietary/hooks/usePolicies.ts b/frontend/editor/src/proprietary/hooks/usePolicies.ts index 356afa3164..be95079b3a 100644 --- a/frontend/editor/src/proprietary/hooks/usePolicies.ts +++ b/frontend/editor/src/proprietary/hooks/usePolicies.ts @@ -6,7 +6,7 @@ * IndexedDB backing folder still holds the editable automation + run state. */ -import { useState, useEffect, useCallback } from "react"; +import { useState, useEffect, useCallback, useRef } from "react"; import { useAppConfig } from "@app/contexts/AppConfigContext"; import { useSaaSTeam } from "@app/contexts/SaaSTeamContext"; import { @@ -14,6 +14,7 @@ import { onPoliciesChange, updatePolicy, resetPolicy, + reorderPolicies as persistPolicyOrder, } from "@app/services/policyStorage"; import { loadPolicyCatalog } from "@app/services/policyCatalog"; import { @@ -33,6 +34,9 @@ import { setPolicyEnabled, removePolicy, } from "@app/services/policyBackend"; +import { reorderPolicies as reorderBackendPolicies } from "@app/services/policyApi"; +import { seedTeamLabelsIfEmpty } from "@app/services/labelsBackend"; +import { getPolicyToolChain } from "@app/components/policies/policyToolChains"; import type { PolicyToStore } from "@app/services/policyPipeline"; import type { PoliciesByCategory, @@ -40,6 +44,12 @@ import type { PolicyWizardResult, } from "@app/types/policies"; +/** Cold-start reconcile retry budget + capped backoff (β‰ˆ0.5sβ†’5s, ~1 min total), + * enough to outlast a backend that starts a little after the frontend. */ +const RECONCILE_MAX_ATTEMPTS = 15; +const reconcileRetryDelay = (attempt: number) => + Math.min(500 * 2 ** attempt, 5000); + /** Build the backend store-request for a category from a wizard result. */ function toStoreRequest( categoryId: string, @@ -65,23 +75,31 @@ function toStoreRequest( export function usePolicies() { const [policies, setPolicies] = useState(loadPolicies); - const { config } = useAppConfig(); + const { config, refetch: refetchAppConfig } = useAppConfig(); const { isTeamLeader } = useSaaSTeam(); useEffect(() => onPoliciesChange(() => setPolicies(loadPolicies())), []); - // Reconcile the local cache against the backend (the source of truth) on - // mount. Backend config wins; the locally-cached folderId (which the backend - // doesn't track) is preserved. If the backend is unreachable we keep the - // local cache as-is, so the surface still works offline. + // Latest refetch, read from inside the retry loop without re-triggering it. + const refetchAppConfigRef = useRef(refetchAppConfig); + refetchAppConfigRef.current = refetchAppConfig; + + // Reconcile local cache against the backend (source of truth), preserving the + // locally-cached folderId; retry with backoff since the backend may not be up yet. + // On recovery, also re-resolve app config in case its admin/team-leader flags settled false while down. useEffect(() => { let cancelled = false; - void (async () => { + let attempt = 0; + let timer: ReturnType | undefined; + + const reconcile = async () => { let byCategory; try { byCategory = await fetchPoliciesByCategory(); } catch { - return; // offline / backend down β€” local cache stands. + if (cancelled || attempt >= RECONCILE_MAX_ATTEMPTS) return; + timer = setTimeout(reconcile, reconcileRetryDelay(attempt++)); + return; } if (cancelled) return; const local = loadPolicies(); @@ -90,14 +108,24 @@ export function usePolicies() { const decoded = byCategory.get(cat.id); reconciled[cat.id] = decoded ? decodedToState(decoded, local[cat.id]?.folderId) - : { ...local[cat.id], configured: false, status: "default" }; + : { + ...local[cat.id], + configured: false, + status: "default", + backendId: undefined, + }; } for (const [id, state] of Object.entries(reconciled)) { updatePolicy(id, state); } - })(); + // The backend was down at first load (we retried) β€” re-resolve the app + // config so the admin/team-leader gate isn't stuck on its offline default. + if (attempt > 0) void refetchAppConfigRef.current(); + }; + void reconcile(); return () => { cancelled = true; + if (timer) clearTimeout(timer); }; }, []); @@ -111,6 +139,13 @@ export function usePolicies() { async (id: string, result: PolicyWizardResult) => { const category = loadPolicyCatalog().categories.find((c) => c.id === id); if (!category) throw new Error(`Unknown policy category: ${id}`); + // A policy whose chain classifies runs against the team's stored label set + // (the engine has no default). Seed it with the built-in defaults now so + // the very first enforced file has a vocabulary to classify against; no-op + // if the team already has a set (never clobbers admin edits). + if (getPolicyToolChain(id)?.includes("classify")) { + await seedTeamLabelsIfEmpty(); + } // One policy per category, ever: reuse any existing backend record. const existingBackendId = loadPolicies()[id]?.backendId ?? @@ -190,6 +225,12 @@ export function usePolicies() { async (id: string, result: PolicyConfigResult) => { const category = loadPolicyCatalog().categories.find((c) => c.id === id); if (!category) throw new Error(`Unknown policy category: ${id}`); + // Seed the team's default label set on first classification-policy setup + // (see enablePolicy) β€” the engine has no default, so the stored set is the + // only vocabulary. No-op once the team has any set. + if (getPolicyToolChain(id)?.includes("classify")) { + await seedTeamLabelsIfEmpty(); + } const current = loadPolicies()[id]; // One policy per category, ever: reuse the existing backend record (even // if the local link was lost) so a save never creates a duplicate. @@ -245,14 +286,42 @@ export function usePolicies() { const pausePolicy = useCallback(async (id: string) => { const current = loadPolicies()[id]; - if (current?.backendId) await setPolicyEnabled(current.backendId, false); + if (current?.backendId) { + await setPolicyEnabled(current.backendId, false).catch((err: unknown) => { + if ( + (err as { response?: { status?: number } })?.response?.status === 404 + ) { + updatePolicy(id, { + backendId: undefined, + configured: false, + status: "default", + }); + return; + } + throw err; + }); + } if (current?.folderId) await setPolicyFolderPaused(current.folderId, true); updatePolicy(id, { status: "paused" }); }, []); const resumePolicy = useCallback(async (id: string) => { const current = loadPolicies()[id]; - if (current?.backendId) await setPolicyEnabled(current.backendId, true); + if (current?.backendId) { + await setPolicyEnabled(current.backendId, true).catch((err: unknown) => { + if ( + (err as { response?: { status?: number } })?.response?.status === 404 + ) { + updatePolicy(id, { + backendId: undefined, + configured: false, + status: "default", + }); + return; + } + throw err; + }); + } if (current?.folderId) await setPolicyFolderPaused(current.folderId, false); updatePolicy(id, { status: "active" }); }, []); @@ -264,6 +333,26 @@ export function usePolicies() { resetPolicy(id); }, []); + /** + * Persist a new execution order for the given categories (in the sequence + * provided). The order is server-side and team-wide: it's mirrored to the + * backend (mapping each category to its stored policy id) so it survives a + * cleared browser and is shared by the whole team. The local cache is updated + * first for an instant re-render; the next reconcile re-reads the server order. + */ + const reorderPolicies = useCallback((orderedCategoryIds: string[]) => { + persistPolicyOrder(orderedCategoryIds); + const current = loadPolicies(); + const backendIds = orderedCategoryIds + .map((categoryId) => current[categoryId]?.backendId) + .filter((id): id is string => !!id); + if (backendIds.length > 0) { + // Fire-and-forget: on failure (offline / not a team leader) the optimistic + // local order stands until the next reconcile re-reads the server's order. + void reorderBackendPolicies(backendIds).catch(() => {}); + } + }, []); + /** * Ensure a configured policy has a *valid* backing folder (its editable * pipeline) and return its id. Self-heals a stale `folderId` β€” one that no @@ -315,6 +404,7 @@ export function usePolicies() { pausePolicy, resumePolicy, deletePolicy, + reorderPolicies, ensurePolicyFolder, }; } diff --git a/frontend/editor/src/proprietary/hooks/usePolicyCatalog.ts b/frontend/editor/src/proprietary/hooks/usePolicyCatalog.ts index 9d2bfbfa2c..c233333e39 100644 --- a/frontend/editor/src/proprietary/hooks/usePolicyCatalog.ts +++ b/frontend/editor/src/proprietary/hooks/usePolicyCatalog.ts @@ -1,4 +1,5 @@ import { useMemo } from "react"; +import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled"; import { loadPolicyCatalog, type PolicyCatalog, @@ -10,7 +11,20 @@ import { * directly. Memoised; when the catalog becomes a backend fetch, this hook is * where loading/error state would be introduced β€” its consumers already treat * it as the single source of definitions. + * + * Categories flagged {@link PolicyCategory.requiresAiEngine} are hidden while the + * AI engine is off, so a policy only appears where it can actually run. */ export function usePolicyCatalog(): PolicyCatalog { - return useMemo(() => loadPolicyCatalog(), []); + const aiEngineEnabled = useAiEngineEnabled(); + return useMemo(() => { + const catalog = loadPolicyCatalog(); + if (aiEngineEnabled) return catalog; + return { + ...catalog, + categories: catalog.categories.filter( + (category) => !category.requiresAiEngine, + ), + }; + }, [aiEngineEnabled]); } diff --git a/frontend/editor/src/proprietary/hooks/useWatchedFolderUrlSync.ts b/frontend/editor/src/proprietary/hooks/useWatchedFolderUrlSync.ts index 6c6a8029ce..18b028ab8e 100644 --- a/frontend/editor/src/proprietary/hooks/useWatchedFolderUrlSync.ts +++ b/frontend/editor/src/proprietary/hooks/useWatchedFolderUrlSync.ts @@ -12,6 +12,7 @@ import { } from "@app/contexts/NavigationContext"; import { useToolWorkflow } from "@app/contexts/ToolWorkflowContext"; import { useAllWatchedFolders } from "@app/hooks/useAllWatchedFolders"; +import { slugify } from "@app/utils/slug"; // Inlined to avoid circular imports β€” must match WatchedFoldersRegistration.tsx const WATCHED_FOLDER_VIEW_ID = "watchedFolder"; @@ -20,13 +21,7 @@ const WATCHED_FOLDER_WORKBENCH_ID = "custom:watchedFolder"; const WATCHED_FOLDERS_BASE = "/watch-folders"; export function slugifyFolderName(name: string): string { - return ( - name - .toLowerCase() - .trim() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-|-$/g, "") || "folder" - ); + return slugify(name) || "folder"; } function parseWatchedFolderRoute(): { diff --git a/frontend/editor/src/proprietary/portal/usersCapabilities.ts b/frontend/editor/src/proprietary/portal/usersCapabilities.ts new file mode 100644 index 0000000000..f8efefc810 --- /dev/null +++ b/frontend/editor/src/proprietary/portal/usersCapabilities.ts @@ -0,0 +1,24 @@ +import type { UsersCapabilities } from "@portal/api/usersCapabilities"; + +/** + * Self-hosted (proprietary) build: org-admin scope - the full set. Matches the + * behaviour the Users page had before the flavor seam existed. + */ +export const usersCapabilities: UsersCapabilities = { + orgGroup: true, + changeRole: true, + adminRole: true, + createTeam: true, + deleteTeam: true, + renameTeam: true, + emailInvite: true, + directCreate: true, + resetPassword: true, + unlock: true, + resetMfa: true, + suspend: true, + moveTeam: true, + seats: false, + manageGrants: true, + removeScope: "org", +}; diff --git a/frontend/editor/src/proprietary/routes/adminRouteExtensions.tsx b/frontend/editor/src/proprietary/routes/adminRouteExtensions.tsx index 89676d2d29..7f60fbdd3b 100644 --- a/frontend/editor/src/proprietary/routes/adminRouteExtensions.tsx +++ b/frontend/editor/src/proprietary/routes/adminRouteExtensions.tsx @@ -1,6 +1,7 @@ import { lazy } from "react"; import type { ReactElement } from "react"; import { Route } from "react-router-dom"; +import { PORTAL_BASENAME } from "@app/routes/portalBasename"; // The portal ships as a lazy chunk of the editor. It's included in dev (so it's // always available to work on) and in production builds made with @@ -8,28 +9,30 @@ import { Route } from "react-router-dom"; // GHA when the portal or AI layers change). Vite replaces the env with a literal at // build time, so when it's off the dynamic import below is tree-shaken out and the // portal chunk isn't emitted. PortalApp stays module-level so it isn't recreated on -// each render. Mocks start first so the worker is ready before the portal's first -// fetch. +// each render. const includePortal = import.meta.env.VITE_INCLUDE_PORTAL === "true" || import.meta.env.DEV; const PortalApp = includePortal ? lazy(async () => { - const { startPortalMocksIfEnabled } = - await import("@portal/mocks/startIfEnabled"); - await startPortalMocksIfEnabled(); const m = await import("@portal/PortalApp"); return { default: m.PortalApp }; }) : null; /** - * The portal mounts as an admin-only route-set at /portal/*. Access is gated - * inside PortalApp (its own AuthProvider + AuthGate, plus server enforcement), - * so this just wires the lazy route into the editor's router when the portal is - * included in this build. + * The portal mounts as an admin-only route-set at PORTAL_BASENAME (/processor/*). + * Access is gated inside PortalApp (its own AuthProvider + AuthGate, plus server + * enforcement), so this just wires the lazy route into the editor's router when + * the portal is included in this build. */ export function getAdminRouteExtensions(): ReactElement[] { if (!PortalApp) return []; - return [} />]; + return [ + } + />, + ]; } diff --git a/frontend/editor/src/proprietary/services/fileSidebarCategories.ts b/frontend/editor/src/proprietary/services/fileSidebarCategories.ts new file mode 100644 index 0000000000..0ae8dac0db --- /dev/null +++ b/frontend/editor/src/proprietary/services/fileSidebarCategories.ts @@ -0,0 +1,196 @@ +// Device-local (localStorage) category structure for the Files sidebar: which parent +// categories exist, their name/icon/order, and which labels roll up into each. It's an editable +// override of the built-in LABEL_FAMILIES default β€” until the user customizes it, the default is +// used verbatim (so the store stays empty and the default can evolve). A label may sit in more +// than one category (multi-membership); a category with `hidden` set stays defined but isn't shown +// as a sidebar group. This is presentational only β€” the classifier never sees categories. + +import { LABEL_FAMILIES } from "@app/data/classificationLabels"; + +export interface SidebarCategory { + /** Stable id β€” built-ins reuse their family id; custom ones get `custom:`. */ + id: string; + name: string; + icon: string; + /** Label ids in this category (matches a file's stored classification ids). */ + labelKeys: string[]; + /** Defined but not rendered as a sidebar group. */ + hidden?: boolean; +} + +// v2: labelKeys hold label ids (was lower-cased names in v1); bumping discards +// stale name-keyed prefs so they don't silently stop matching. +const STORAGE_KEY = "stirling.fileSidebarCategories.v2"; + +/** The built-in default, derived from LABEL_FAMILIES. Fresh copy per call (callers may mutate). */ +export function defaultCategories(): SidebarCategory[] { + return LABEL_FAMILIES.map((family) => ({ + id: family.id, + name: family.name, + icon: family.icon, + labelKeys: family.labels.map((label) => label.id), + })); +} + +function readStorage(): SidebarCategory[] | null { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (!raw) return null; + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) return null; + return parsed + .filter((c): c is SidebarCategory => { + const cat = c as Partial; + return ( + typeof cat.id === "string" && + typeof cat.name === "string" && + typeof cat.icon === "string" && + Array.isArray(cat.labelKeys) + ); + }) + .map((c) => ({ + id: c.id, + name: c.name, + icon: c.icon, + labelKeys: c.labelKeys.filter((k) => typeof k === "string"), + hidden: c.hidden === true, + })); + } catch { + return null; + } +} + +// null = using the built-in default (not yet customized). Cached so useSyncExternalStore sees a +// stable reference between writes. +let stored: SidebarCategory[] | null = readStorage(); +const listeners = new Set<() => void>(); + +// Effective list, recomputed only on write so its identity is stable for memo/useSyncExternalStore. +let effective: SidebarCategory[] = stored ?? defaultCategories(); + +function recompute() { + effective = stored ?? defaultCategories(); +} + +function write(next: SidebarCategory[]) { + stored = next; + recompute(); + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify(next)); + } catch { + // Quota/private-mode failures degrade to session-only categories. + } + for (const listener of listeners) listener(); +} + +/** Mutate the current effective list (snapshotting the default on first edit). */ +function mutate(fn: (categories: SidebarCategory[]) => SidebarCategory[]) { + write(fn(effective.map((c) => ({ ...c, labelKeys: [...c.labelKeys] })))); +} + +export function getSidebarCategories(): SidebarCategory[] { + return effective; +} + +export function subscribeSidebarCategories(listener: () => void) { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +export function isCustomized(): boolean { + return stored !== null; +} + +/** Map each label key to the ids of every VISIBLE category it belongs to. */ +export function labelCategoryMap( + categories: SidebarCategory[], +): Map { + const map = new Map(); + for (const category of categories) { + if (category.hidden) continue; + for (const key of category.labelKeys) { + const ids = map.get(key); + if (ids) ids.push(category.id); + else map.set(key, [category.id]); + } + } + return map; +} + +/** Set of every label key that belongs to any category (visible or not). */ +export function categorizedLabelKeys( + categories: SidebarCategory[], +): Set { + const keys = new Set(); + for (const category of categories) { + for (const key of category.labelKeys) keys.add(key); + } + return keys; +} + +// ---- editing ---- + +/** Create a new empty category; returns its id. */ +export function addCategory(name: string, icon: string): string { + const id = `custom:${name.toLowerCase().replace(/[^a-z0-9]+/g, "-")}-${ + effective.length + }`; + mutate((categories) => [...categories, { id, name, icon, labelKeys: [] }]); + return id; +} + +export function renameCategory(id: string, name: string) { + mutate((categories) => + categories.map((c) => (c.id === id ? { ...c, name } : c)), + ); +} + +export function setCategoryIcon(id: string, icon: string) { + mutate((categories) => + categories.map((c) => (c.id === id ? { ...c, icon } : c)), + ); +} + +export function setCategoryHidden(id: string, hidden: boolean) { + mutate((categories) => + categories.map((c) => (c.id === id ? { ...c, hidden } : c)), + ); +} + +export function deleteCategory(id: string) { + mutate((categories) => categories.filter((c) => c.id !== id)); +} + +export function addLabelToCategory(id: string, labelId: string) { + mutate((categories) => + categories.map((c) => + c.id === id && !c.labelKeys.includes(labelId) + ? { ...c, labelKeys: [...c.labelKeys, labelId] } + : c, + ), + ); +} + +export function removeLabelFromCategory(id: string, labelId: string) { + mutate((categories) => + categories.map((c) => + c.id === id + ? { ...c, labelKeys: c.labelKeys.filter((k) => k !== labelId) } + : c, + ), + ); +} + +/** Restore the built-in default and clear the customized flag (undoes any prior `mutate`). */ +export function resetSidebarCategories() { + stored = null; + recompute(); + try { + localStorage.removeItem(STORAGE_KEY); + } catch { + // Quota/private-mode failures degrade to session-only categories. + } + for (const listener of listeners) listener(); +} diff --git a/frontend/editor/src/proprietary/services/labelsBackend.test.ts b/frontend/editor/src/proprietary/services/labelsBackend.test.ts new file mode 100644 index 0000000000..d530364b3a --- /dev/null +++ b/frontend/editor/src/proprietary/services/labelsBackend.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import apiClient from "@app/services/apiClient"; +import { DEFAULT_CLASSIFICATION_LABELS } from "@app/data/classificationLabels"; +import { seedTeamLabelsIfEmpty } from "@app/services/labelsBackend"; + +vi.mock("@app/services/apiClient"); + +const get = vi.mocked(apiClient.get); +const put = vi.mocked(apiClient.put); + +// The service only reads `status`/`data` off the axios response; a partial shape +// is all these tests need, so cast the mock values through this helper. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const res = (value: object): any => value; + +// 204 No Content (nothing stored) comes back as an empty body. +const emptyResponse = res({ status: 204, data: "" }); +const storedResponse = res({ + status: 200, + data: { labels: [{ id: "invoice", name: "Invoice", icon: "receipt" }] }, +}); +const putEcho = res({ data: { labels: DEFAULT_CLASSIFICATION_LABELS } }); + +describe("seedTeamLabelsIfEmpty", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("seeds the built-in defaults when the team has no set", async () => { + get.mockResolvedValue(emptyResponse); + put.mockResolvedValue(putEcho); + + await seedTeamLabelsIfEmpty(); + + expect(put).toHaveBeenCalledTimes(1); + expect(put).toHaveBeenCalledWith("/api/v1/classification/labels", { + labels: DEFAULT_CLASSIFICATION_LABELS, + }); + }); + + it("is a no-op when the team already has a set (never clobbers)", async () => { + get.mockResolvedValue(storedResponse); + + await seedTeamLabelsIfEmpty(); + + expect(put).not.toHaveBeenCalled(); + }); + + it("rides out a transient fetch failure, then seeds", async () => { + vi.useFakeTimers(); + get + .mockRejectedValueOnce(new Error("network blip")) + .mockResolvedValueOnce(emptyResponse); + put.mockResolvedValue(putEcho); + + const pending = seedTeamLabelsIfEmpty(); + await vi.runAllTimersAsync(); + await pending; + + expect(get).toHaveBeenCalledTimes(2); + expect(put).toHaveBeenCalledTimes(1); + }); + + it("rides out a transient write failure, then seeds", async () => { + vi.useFakeTimers(); + get.mockResolvedValue(emptyResponse); + put + .mockRejectedValueOnce(new Error("network blip")) + .mockResolvedValueOnce(putEcho); + + const pending = seedTeamLabelsIfEmpty(); + await vi.runAllTimersAsync(); + await pending; + + expect(put).toHaveBeenCalledTimes(2); + }); + + it("throws after a persistent fetch failure without writing (clobber-safe)", async () => { + vi.useFakeTimers(); + get.mockRejectedValue(new Error("backend down")); + + const pending = seedTeamLabelsIfEmpty(); + const assertion = expect(pending).rejects.toThrow("backend down"); + await vi.runAllTimersAsync(); + await assertion; + + // Never writes when the fetch can't confirm the team has no set. + expect(put).not.toHaveBeenCalled(); + expect(get).toHaveBeenCalledTimes(3); + }); + + it("throws after a persistent write failure", async () => { + vi.useFakeTimers(); + get.mockResolvedValue(emptyResponse); + put.mockRejectedValue(new Error("write failed")); + + const pending = seedTeamLabelsIfEmpty(); + const assertion = expect(pending).rejects.toThrow("write failed"); + await vi.runAllTimersAsync(); + await assertion; + + expect(put).toHaveBeenCalledTimes(3); + }); +}); diff --git a/frontend/editor/src/proprietary/services/labelsBackend.ts b/frontend/editor/src/proprietary/services/labelsBackend.ts new file mode 100644 index 0000000000..8259f124de --- /dev/null +++ b/frontend/editor/src/proprietary/services/labelsBackend.ts @@ -0,0 +1,99 @@ +/** + * Backend layer for the team's classification labels + * (`/api/v1/classification/labels`) β€” one shared, server-truth list, editable + * only by a team leader (SaaS) / admin (self-hosted). A team with none reads as + * 204 β†’ `null`, and callers fall back to the built-in + * {@link DEFAULT_CLASSIFICATION_LABELS}. + */ + +import apiClient from "@app/services/apiClient"; +import { + DEFAULT_CLASSIFICATION_LABELS, + type ClassificationLabel, +} from "@app/data/classificationLabels"; + +const TEAM_ENDPOINT = "/api/v1/classification/labels"; + +/** Wire shape shared with the backend: the label list wrapped in an object. */ +interface LabelsPayload { + labels: ClassificationLabel[]; +} + +async function fetchLabels( + endpoint: string, +): Promise { + const res = await apiClient.get(endpoint, { + suppressErrorToast: true, + }); + // 204 No Content (nothing stored) comes back as an empty body. Only an + // explicit 204 / empty string means "none"; anything else is a real payload. + if (res.status === 204 || res.data === "") return null; + return (res.data as LabelsPayload).labels ?? []; +} + +async function saveLabels( + endpoint: string, + labels: ClassificationLabel[], +): Promise { + const res = await apiClient.put(endpoint, { labels }); + return res.data.labels ?? []; +} + +/** The team's stored labels, or `null` when it has none (use the default). */ +export function fetchTeamLabels(): Promise { + return fetchLabels(TEAM_ENDPOINT); +} + +/** Persist the team's labels; returns the stored value. */ +export function saveTeamLabels( + labels: ClassificationLabel[], +): Promise { + return saveLabels(TEAM_ENDPOINT, labels); +} + +/** Seed-write retry budget: the seed is a hard prerequisite (see below), so ride + * out a transient blip rather than fail setup on the first hiccup. */ +const SEED_MAX_ATTEMPTS = 3; +const SEED_RETRY_MS = 400; + +async function withRetry(op: () => Promise): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < SEED_MAX_ATTEMPTS; attempt++) { + try { + return await op(); + } catch (error) { + lastError = error; + if (attempt < SEED_MAX_ATTEMPTS - 1) { + await new Promise((resolve) => + setTimeout(resolve, SEED_RETRY_MS * (attempt + 1)), + ); + } + } + } + throw lastError; +} + +/** + * Seed the team's label set with the built-in defaults when it has none yet. + * + * The engine holds no default vocabulary, so the backend can only send what the + * team has stored β€” a team with an empty set classifies nothing (documents come + * back unlabelled and don't group). This writes the frontend's single default + * copy into the team set the first time a classification policy is set up, so + * classification works out of the box; the backend gates the write to admins. + * + * The seed is a hard prerequisite of enabling a classification policy, so it must + * land or fail loudly β€” never silently leave an empty set behind a created + * policy. Both the fetch and the write retry a transient failure; if either + * ultimately fails this THROWS, and the caller aborts the setup (the admin sees + * the error and retries) instead of shipping a policy that classifies nothing β€” + * the same way a failed policy save already aborts setup. Still clobber-safe: it + * writes only when the fetch DEFINITIVELY reports no set (204 β†’ null), so it + * never overwrites a team's real (possibly customised) labels, and it's a no-op + * once any set exists (later admin edits are the source of truth). + */ +export async function seedTeamLabelsIfEmpty(): Promise { + const existing = await withRetry(fetchTeamLabels); + if (existing != null) return; + await withRetry(() => saveTeamLabels(DEFAULT_CLASSIFICATION_LABELS)); +} diff --git a/frontend/editor/src/proprietary/services/labelsFile.ts b/frontend/editor/src/proprietary/services/labelsFile.ts new file mode 100644 index 0000000000..613b144c1f --- /dev/null +++ b/frontend/editor/src/proprietary/services/labelsFile.ts @@ -0,0 +1,107 @@ +/** + * Client-side import/export + validation for a classification-labels JSON file + * (`{"labels":[{"id":"invoice","name":"Invoice","icon":"receipt-long"}, …]}`). + * Sharing a label set between teams is done by exporting the JSON here and + * importing it on another team. Validation mirrors the backend + * (`LabelsValidator`) so a malformed file is caught before it's uploaded β€” the + * backend re-validates as the authority. A file may omit `id` (e.g. an older + * export); it is then derived from the name. + */ + +import { downloadJsonAsFile } from "@app/utils/downloadUtils"; +import { LABEL_ICON_KEYS } from "@app/data/labelIcons"; +import { + labelId, + type ClassificationLabel, +} from "@app/data/classificationLabels"; + +// Kept in sync with the backend LabelsValidator (the authority); enforced here +// too so an oversized import is rejected before upload. +const MAX_LABELS = 500; +const MAX_TEXT_LENGTH = 128; + +interface LabelsFileShape { + labels: ClassificationLabel[]; +} + +/** A label's effective id: the provided one, else derived from the name. */ +function effectiveId(label: Partial): string { + const provided = typeof label.id === "string" ? label.id.trim() : ""; + return provided || labelId(label.name?.trim() ?? ""); +} + +/** Human-readable problems with a candidate labels file; empty means valid. */ +export function validateLabels(value: unknown): string[] { + const errors: string[] = []; + if (typeof value !== "object" || value === null) { + return ["File is not a labels object."]; + } + const { labels } = value as Partial; + if (!Array.isArray(labels)) { + return ['File must have a "labels" list.']; + } + if (labels.length > MAX_LABELS) { + errors.push(`Too many labels (max ${MAX_LABELS}).`); + } + const seenIds = new Set(); + for (const label of labels) { + if (!isText(label?.name)) { + errors.push("Every label needs a non-empty name."); + continue; + } + const name = label.name.trim(); + if (name.length > MAX_TEXT_LENGTH) { + errors.push(`Label "${name}" is over ${MAX_TEXT_LENGTH} characters.`); + } + const id = effectiveId(label); + if (!id) { + errors.push(`Label "${name}" has no usable id.`); + continue; + } + if (seenIds.has(id)) errors.push(`Duplicate label: ${name}`); + seenIds.add(id); + } + return errors; +} + +/** Coerce a validated value into a normalized label list (trims, drops extras). */ +export function normalizeLabels( + labels: ClassificationLabel[], +): ClassificationLabel[] { + return labels.map((label): ClassificationLabel => { + const name = label.name.trim(); + const id = effectiveId(label); + // Keep the icon only if it's a known palette key, so a hand-crafted import + // can't set an unbundled key that renders blank. + return label.icon && LABEL_ICON_KEYS.has(label.icon) + ? { id, name, icon: label.icon } + : { id, name }; + }); +} + +/** Parse + validate a picked file, resolving to a normalized label list. */ +export async function parseLabelsFile( + file: File, +): Promise { + let parsed: unknown; + try { + parsed = JSON.parse(await file.text()); + } catch { + throw new Error("That file isn't valid JSON."); + } + const errors = validateLabels(parsed); + if (errors.length > 0) throw new Error(errors[0]); + return normalizeLabels((parsed as LabelsFileShape).labels); +} + +/** Trigger a download of the labels as a pretty-printed JSON file. */ +export function downloadLabels( + labels: ClassificationLabel[], + fileName = "classification-labels.json", +): void { + downloadJsonAsFile({ labels }, fileName); +} + +function isText(value: unknown): value is string { + return typeof value === "string" && value.trim().length > 0; +} diff --git a/frontend/editor/src/proprietary/services/policyApi.ts b/frontend/editor/src/proprietary/services/policyApi.ts index b3f1fd999f..a10c7a8382 100644 --- a/frontend/editor/src/proprietary/services/policyApi.ts +++ b/frontend/editor/src/proprietary/services/policyApi.ts @@ -51,6 +51,15 @@ export async function deletePolicy(id: string): Promise { await apiClient.delete(`/api/v1/policies/${encodeURIComponent(id)}`); } +/** + * Persist the team's run order (server-side, shared by the whole team). Sends the + * ordered backend policy ids; the backend maps position β†’ order and ignores any + * id outside the caller's team. Team-leader/admin only (403 otherwise). + */ +export async function reorderPolicies(orderedIds: string[]): Promise { + await apiClient.put("/api/v1/policies/order", orderedIds); +} + /** Run a stored policy by id on the supplied files; returns the run id. */ export async function runStoredPolicy( id: string, diff --git a/frontend/editor/src/proprietary/services/policyBackend.ts b/frontend/editor/src/proprietary/services/policyBackend.ts index e1ee2400f4..fdec83494d 100644 --- a/frontend/editor/src/proprietary/services/policyBackend.ts +++ b/frontend/editor/src/proprietary/services/policyBackend.ts @@ -26,12 +26,15 @@ import type { PolicyState } from "@app/types/policies"; export async function fetchPoliciesByCategory(): Promise< Map > { + // The backend returns policies in the team's run order; the list index IS the + // order (server-side, shared team-wide), which we carry onto the decoded state. const stored = await policyApi.listPolicies(); const byCategory = new Map(); - for (const policy of stored) { + stored.forEach((policy, index) => { const decoded = fromBackendPolicy(policy); - if (decoded.categoryId) byCategory.set(decoded.categoryId, decoded); - } + if (decoded.categoryId) + byCategory.set(decoded.categoryId, { ...decoded, order: index }); + }); return byCategory; } @@ -57,6 +60,8 @@ export function decodedToState( runOn: decoded.folder.runOn, folderId: localFolderId, backendId: decoded.id, + // Server-side run-order position (team-wide); drives the settings reorder list. + order: decoded.order, // Catalog-category policies are built-in defaults (not deletable). isDefault: true, }; diff --git a/frontend/editor/src/proprietary/services/policyLiveData.test.ts b/frontend/editor/src/proprietary/services/policyLiveData.test.ts index c3016f071b..1bb4687164 100644 --- a/frontend/editor/src/proprietary/services/policyLiveData.test.ts +++ b/frontend/editor/src/proprietary/services/policyLiveData.test.ts @@ -4,6 +4,8 @@ import { runsToActivity, runsToStats, policyActiveFor, + progressByCategory, + retryableFailedRuns, } from "@app/services/policyLiveData"; import type { PolicyRunRecord } from "@app/components/policies/policyRunStore"; @@ -120,6 +122,117 @@ describe("runsToStats", () => { }); }); +describe("progressByCategory", () => { + it("tallies running / completed / total per category", () => { + const map = progressByCategory([ + run({ runId: "a", categoryId: "classification", status: "RUNNING" }), + run({ + runId: "b", + categoryId: "classification", + status: "COMPLETED", + imported: true, + }), + run({ runId: "c", categoryId: "security", status: "RUNNING" }), + ]); + expect(map.get("classification")).toEqual({ + running: 1, + completed: 1, + total: 2, + }); + expect(map.get("security")).toEqual({ running: 1, completed: 0, total: 1 }); + }); + + it("a COMPLETED-but-not-imported run still counts as running (not done)", () => { + const map = progressByCategory([ + run({ categoryId: "security", status: "COMPLETED", imported: false }), + ]); + expect(map.get("security")).toEqual({ running: 1, completed: 0, total: 1 }); + }); + + it("scopes to the current wave β€” history before sinceStartedAt is excluded", () => { + // The store persists runs across every upload; the panel must only count the + // current wave, so an old batch (startedAt < sinceStartedAt) is ignored. + const map = progressByCategory( + [ + run({ runId: "new1", status: "RUNNING", startedAt: 1000 }), + run({ + runId: "new2", + status: "COMPLETED", + imported: true, + startedAt: 1000, + }), + run({ + runId: "old", + status: "COMPLETED", + imported: true, + startedAt: 10, + }), + ], + 1000, + ); + // Only the two wave runs counted; the pre-wave "old" run is excluded. + expect(map.get("security")).toEqual({ running: 1, completed: 1, total: 2 }); + }); +}); + +describe("retryableFailedRuns β€” bulk-retry eligibility", () => { + it("includes a plain failed run", () => { + const eligible = retryableFailedRuns([ + run({ runId: "a", status: "FAILED" }), + ]); + expect(eligible.map((r) => r.runId)).toEqual(["a"]); + }); + + it("NEVER retries a (policy, file) that has since succeeded", () => { + // The caveat: a stale failure row must not re-enforce a file that already + // went through β€” e.g. the user retried it individually and it completed. + const eligible = retryableFailedRuns([ + run({ runId: "ok", status: "COMPLETED", imported: true }), // newest + run({ runId: "old-fail", status: "FAILED" }), + ]); + expect(eligible).toEqual([]); + }); + + it("skips a (policy, file) whose retry is already running", () => { + const eligible = retryableFailedRuns([ + run({ runId: "live", status: "RUNNING" }), // newest + run({ runId: "old-fail", status: "FAILED" }), + ]); + expect(eligible).toEqual([]); + }); + + it("keeps only the LATEST failed attempt per (policy, file)", () => { + const eligible = retryableFailedRuns([ + run({ runId: "fail-new", status: "FAILED", startedAt: 2 }), + run({ runId: "fail-old", status: "FAILED", startedAt: 1 }), + ]); + expect(eligible.map((r) => r.runId)).toEqual(["fail-new"]); + }); + + it("treats policies independently β€” the same file can retry under another policy", () => { + const eligible = retryableFailedRuns([ + run({ runId: "sec-ok", categoryId: "security", status: "COMPLETED" }), + run({ runId: "wm-fail", categoryId: "watermark", status: "FAILED" }), + ]); + expect(eligible.map((r) => r.runId)).toEqual(["wm-fail"]); + }); + + it("excludes runs mid auto-retry backoff and orphans with no local file", () => { + const eligible = retryableFailedRuns([ + run({ runId: "backoff", status: "FAILED", retrying: true }), + run({ runId: "orphan", fileId: "", status: "FAILED" }), + ]); + expect(eligible).toEqual([]); + }); + + it("includes cancelled runs (they never produced output)", () => { + const eligible = retryableFailedRuns([ + run({ runId: "c", status: "CANCELLED" }), + ]); + expect(eligible.map((r) => r.runId)).toEqual(["c"]); + }); +}); + describe("policyActiveFor", () => { it("returns 'Today' for a just-activated policy", () => { expect(policyActiveFor(new Date().toISOString())).toBe("Today"); diff --git a/frontend/editor/src/proprietary/services/policyLiveData.ts b/frontend/editor/src/proprietary/services/policyLiveData.ts index 56401f4d19..2f177bb987 100644 --- a/frontend/editor/src/proprietary/services/policyLiveData.ts +++ b/frontend/editor/src/proprietary/services/policyLiveData.ts @@ -7,7 +7,92 @@ import i18n from "@app/i18n"; import type { PolicyActivityItem, PolicyStats } from "@app/types/policies"; -import type { PolicyRunRecord } from "@app/components/policies/policyRunStore"; +import { + dispatchKey, + isRunInFlight, + type PolicyRunRecord, +} from "@app/components/policies/policyRunStore"; + +/** + * The failed runs that are actually worth re-running, one per (policy, file): + * - terminal-failed (FAILED/CANCELLED) and not already in an auto-retry backoff; + * - with a local input file to re-run on (reconciled server orphans have none); + * - EXCLUDING any (policy, file) that has since succeeded or is currently + * running β€” a stale failure row must never re-enforce a file that already + * went through, or race a run that's still going. + * Runs are newest-first in the store, so the first failed run seen per key is + * the latest attempt. This is the groundwork for bulk "retry all failed" and, + * later, "run a newly-enabled policy across not-yet-processed files". + */ +export function retryableFailedRuns( + runs: PolicyRunRecord[], +): PolicyRunRecord[] { + // Keys settled by a success or still being worked β€” off-limits for retry. + const settledOrActive = new Set(); + for (const run of runs) { + if (run.status === "COMPLETED" || isRunInFlight(run)) { + settledOrActive.add(dispatchKey(run.categoryId, run.fileId)); + } + } + const seen = new Set(); + const eligible: PolicyRunRecord[] = []; + for (const run of runs) { + if (run.status !== "FAILED" && run.status !== "CANCELLED") continue; + if (run.retrying) continue; + if (!run.fileId) continue; + const key = dispatchKey(run.categoryId, run.fileId); + if (settledOrActive.has(key) || seen.has(key)) continue; + seen.add(key); + eligible.push(run); + } + return eligible; +} + +/** Live per-policy run tally β€” drives the panel's processing ring + counts. */ +export interface PolicyRunProgress { + /** Runs still working: dispatched/running, retrying, or done-but-not-imported. */ + running: number; + /** Runs that finished successfully AND landed in the workspace. */ + completed: number; + /** Every run in the current wave for the policy (running + completed + failed). */ + total: number; +} + +/** Empty tally, so callers can render a zero state without null checks. */ +export const EMPTY_RUN_PROGRESS: PolicyRunProgress = { + running: 0, + completed: 0, + total: 0, +}; + +/** + * Tally runs per policy category for the panel's live indicators: how many are + * still processing, how many completed, and the total in the CURRENT wave. + * Keyed by categoryId. + * + * `sinceStartedAt` scopes to the current upload wave (the store resets it when a + * run starts with nothing in flight) so the counts reflect "this upload", not the + * whole run history persisted in localStorage across every past upload. + */ +export function progressByCategory( + runs: PolicyRunRecord[], + sinceStartedAt = 0, +): Map { + const byCat = new Map(); + for (const run of runs) { + if (run.startedAt < sinceStartedAt) continue; + const p = byCat.get(run.categoryId) ?? { + running: 0, + completed: 0, + total: 0, + }; + p.total += 1; + if (isRunInFlight(run)) p.running += 1; + else if (run.status === "COMPLETED") p.completed += 1; + byCat.set(run.categoryId, p); + } + return byCat; +} /** Relative "Nm/Nh ago" for an activity timestamp (epoch ms). */ function relativeTime(ts: number): string { diff --git a/frontend/editor/src/proprietary/services/policyPipeline.ts b/frontend/editor/src/proprietary/services/policyPipeline.ts index b27bd09e4d..c04adb78b6 100644 --- a/frontend/editor/src/proprietary/services/policyPipeline.ts +++ b/frontend/editor/src/proprietary/services/policyPipeline.ts @@ -101,6 +101,16 @@ export interface PolicyRunView { createdAt: number; } +/** + * Operations that run as policy pipeline steps but are NOT user-facing tools, so + * they have no tool-registry entry and never appear in the tool picker. Maps the + * operation id straight to its backend endpoint. + */ +const POLICY_OPERATION_ENDPOINTS: Record = { + // Document classification β€” dispatched only by the Classification policy. + classify: "/api/v1/ai/tools/classify-and-label", +}; + /** Resolve a frontend operation id to its backend tool endpoint path. */ function resolveEndpoint( operation: string, @@ -109,10 +119,13 @@ function resolveEndpoint( ): string | null { const config = toolRegistry[operation as keyof ToolRegistry]?.operationConfig; const endpoint = config?.endpoint; - if (!endpoint) return null; - const resolved = - typeof endpoint === "function" ? endpoint(parameters) : endpoint; - return resolved ?? null; + if (endpoint) { + const resolved = + typeof endpoint === "function" ? endpoint(parameters) : endpoint; + if (resolved) return resolved; + } + // Policy-only operations have no registry entry; resolve them directly. + return POLICY_OPERATION_ENDPOINTS[operation] ?? null; } /** @@ -222,6 +235,9 @@ export interface DecodedPolicy { reviewerEmail: string; fieldValues: Record; folder: PolicyFolderSettings; + /** Position in the team's server-side run order (set from the fetch list index, + * not decoded from the policy itself). */ + order?: number; } const DEFAULT_FOLDER: PolicyFolderSettings = { diff --git a/frontend/editor/src/proprietary/services/policyStorage.ts b/frontend/editor/src/proprietary/services/policyStorage.ts index deed307945..82e90b0d40 100644 --- a/frontend/editor/src/proprietary/services/policyStorage.ts +++ b/frontend/editor/src/proprietary/services/policyStorage.ts @@ -53,14 +53,17 @@ export function loadPolicies(): PoliciesByCategory { // Always reconcile against the current category list so a newly-added // category gets a default rather than being undefined. const out: PoliciesByCategory = {}; - for (const cat of loadPolicyCatalog().categories) { + loadPolicyCatalog().categories.forEach((cat, index) => { const merged = { ...defaultState(), ...(parsed[cat.id] ?? {}) }; // Migration: clear the obsolete persisted reviewer email so it re-defaults // to the real signed-in user. if (merged.reviewerEmail === STALE_REVIEWER_EMAIL) merged.reviewerEmail = ""; + // Default execution order to the catalog position until an admin reorders, + // so ordered dispatch is deterministic before any explicit order is set. + if (merged.order == null) merged.order = index; out[cat.id] = merged; - } + }); return out; } @@ -97,6 +100,24 @@ export function updatePolicy( return next; } +/** + * Persist a new execution order. Assigns `order` 0..n-1 to the given categories in + * the sequence provided, so after any reorder every listed policy has an explicit, + * contiguous order (no reliance on the catalog-index default). Categories omitted + * from the list keep their current order. + */ +export function reorderPolicies( + orderedCategoryIds: string[], +): PoliciesByCategory { + const current = loadPolicies(); + const next: PoliciesByCategory = { ...current }; + orderedCategoryIds.forEach((id, index) => { + if (next[id]) next[id] = { ...next[id], order: index }; + }); + persist(next); + return next; +} + /** Reset a category to its unconfigured default (the "Delete policy" action). */ export function resetPolicy(categoryId: string): PoliciesByCategory { return updatePolicy(categoryId, { diff --git a/frontend/editor/src/proprietary/types/policies.ts b/frontend/editor/src/proprietary/types/policies.ts index b0285a4d22..a1f6ea70f6 100644 --- a/frontend/editor/src/proprietary/types/policies.ts +++ b/frontend/editor/src/proprietary/types/policies.ts @@ -51,6 +51,11 @@ export interface PolicyCategory { * or configured. Only Security is live today. */ comingSoon?: boolean; + /** + * Requires the AI engine to be enabled. Hidden from the catalog when the + * engine is off, so the policy only appears where it can actually run. + */ + requiresAiEngine?: boolean; } /** @@ -136,6 +141,13 @@ export interface PolicyState { outputNamePosition?: "prefix" | "suffix" | "auto-number"; /** When the policy runs: on "upload" or before "export". Defaults to "upload". */ runOn?: "upload" | "export"; + /** + * Execution order among policies that share a trigger. When several policies run + * on the same event they fire in ascending `order`, each on the previous one's + * output (a cumulative chain). Defaults to the policy's position in the catalog + * until an admin reorders them, which persists an explicit value for every policy. + */ + order?: number; /** * The backing folder-trigger record (a Watched Folders `WatchedFolder`) that * holds this policy's editable steps (its automation), output config and run diff --git a/frontend/editor/src/saas/components/shared/AppConfigModal.tsx b/frontend/editor/src/saas/components/shared/AppConfigModal.tsx index b691244e68..7d36b40ee4 100644 --- a/frontend/editor/src/saas/components/shared/AppConfigModal.tsx +++ b/frontend/editor/src/saas/components/shared/AppConfigModal.tsx @@ -9,7 +9,10 @@ import { useTranslation } from "react-i18next"; import LocalIcon from "@app/components/shared/LocalIcon"; import Overview from "@app/components/shared/config/configSections/Overview"; import { createSaasConfigNavSections } from "@app/components/shared/config/saasConfigNavSections"; -import { NavKey } from "@app/components/shared/config/types"; +import { + NavKey, + type ConfigNavSection, +} from "@app/components/shared/config/types"; import { stripBasePath, withBasePath } from "@app/constants/app"; import { COOKIE_CONSENT_SCROLL_SHARD } from "@app/hooks/useCookieConsent"; import "@app/components/shared/AppConfigModal.css"; @@ -21,9 +24,21 @@ import { interface AppConfigModalProps { opened: boolean; onClose: () => void; + /** Accepted for interface parity with the core shell; this shell never + * URL-syncs, so it has no effect. */ + urlSync?: boolean; + /** Section to land on when opening (used by non-URL hosts like the portal). */ + initialSection?: NavKey | null; + /** Host-specific sections appended after the saas registry sections. */ + extraSections?: ConfigNavSection[]; } -const AppConfigModal: React.FC = ({ opened, onClose }) => { +const AppConfigModal: React.FC = ({ + opened, + onClose, + initialSection, + extraSections, +}) => { const isMobile = useMediaQuery("(max-width: 1024px)"); const { signOut, user } = useAuth(); @@ -53,16 +68,21 @@ const AppConfigModal: React.FC = ({ opened, onClose }) => { // usage-limit modal CTAs, which need to land on the Plan section), select that section. The // opener (QuickAccessBar) opens the modal whenever the path is /settings/*, but doesn't carry // the section, and `active` defaults to "overview" β€” so without this a deep link would open on - // Overview rather than the linked section. + // Overview rather than the linked section. Non-URL hosts (the portal) pass the + // section directly instead. useEffect(() => { if (!opened) return; + if (initialSection) { + setActive(initialSection); + return; + } const match = stripBasePath(window.location.pathname).match( /^\/settings\/([^/?#]+)/, ); if (match) { setActive(match[1] as NavKey); } - }, [opened]); + }, [opened, initialSection]); // Listen for notice updates (e.g., "Not enough credits..." next to Plan title) useEffect(() => { @@ -116,15 +136,14 @@ const AppConfigModal: React.FC = ({ opened, onClose }) => { // Left navigation structure and icons. The Plan tab now internally branches // free vs subscribed Γ— leader vs member via useWallet(), so the modal no // longer plumbs paygEnabled / isLeader through to the nav builder. - const configNavSections = useMemo( - () => - createSaasConfigNavSections(Overview, openLogoutConfirm, { - isDev, - isAnonymous, - t, - }), - [openLogoutConfirm, isDev, isAnonymous, t], - ); + const configNavSections = useMemo(() => { + const sections = createSaasConfigNavSections(Overview, openLogoutConfirm, { + isDev, + isAnonymous, + t, + }); + return extraSections?.length ? [...sections, ...extraSections] : sections; + }, [openLogoutConfirm, isDev, isAnonymous, t, extraSections]); const activeLabel = useMemo(() => { for (const section of configNavSections) { diff --git a/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.css b/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.css new file mode 100644 index 0000000000..de669c7f29 --- /dev/null +++ b/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.css @@ -0,0 +1,136 @@ +/* Category-manager modal: editable parent categories with member-label chips. + Matches the labels editor's width so the two read as a set. */ +.fsg-modal { + width: min(1100px, 94vw); + max-width: min(1100px, 94vw); +} + +.fsg-body { + display: flex; + flex-direction: column; + gap: 12px; + max-height: 60vh; + overflow-y: auto; + padding-right: 4px; +} + +.fsg-cat { + display: flex; + flex-direction: column; + gap: 6px; +} + +.fsg-cat-header { + display: flex; + align-items: center; + gap: 6px; + font-size: 13px; + font-weight: 600; + color: var(--text-primary, inherit); +} + +.fsg-cat-toggle { + display: inline-flex; + border: none; + background: transparent; + padding: 0; + cursor: pointer; +} + +.fsg-chevron { + font-size: 1.1rem !important; + color: var(--text-muted, rgba(128, 128, 128, 0.9)); + margin-left: -4px; +} + +.fsg-cat-name { + flex: 1; + text-align: left; + border: none; + background: transparent; + padding: 2px 4px; + border-radius: 4px; + font: inherit; + color: inherit; + cursor: text; +} + +.fsg-cat-name:hover { + background: var(--hover-bg, rgba(128, 128, 128, 0.12)); +} + +.fsg-rename { + flex: 1; +} + +.fsg-cat-action { + display: inline-flex; + border: none; + background: transparent; + padding: 2px; + border-radius: 4px; + cursor: pointer; + color: var(--text-muted, rgba(128, 128, 128, 0.9)); +} + +.fsg-cat-action:hover { + background: var(--hover-bg, rgba(128, 128, 128, 0.12)); + color: var(--text-primary, inherit); +} + +.fsg-cat-body { + display: flex; + flex-direction: column; + gap: 6px; + /* Indent past the chevron so the hierarchy reads at a glance. */ + padding-left: 24px; +} + +/* Member chips are the shared @app/ui/LabelChip. */ +.fsg-chips { + display: flex; + flex-wrap: wrap; + gap: 4px; +} + +.fsg-count { + font-size: 11px; + font-weight: 500; + color: var(--text-muted, rgba(128, 128, 128, 0.9)); +} + +.fsg-add, +.fsg-new { + display: flex; + align-items: center; + gap: 6px; +} + +.fsg-new { + padding-top: 4px; + border-top: 1px solid var(--border-subtle, rgba(128, 128, 128, 0.2)); +} + +.fsg-add-input { + flex: 1; + min-width: 0; + padding: 4px 8px; + font-size: 12px; + border: 1px solid var(--border-subtle, rgba(128, 128, 128, 0.35)); + border-radius: 6px; + background: transparent; + color: var(--text-primary, inherit); +} + +.fsg-add-input:focus { + outline: none; + border-color: var(--color-primary, #4f46e5); +} + +.fsg-footer { + display: flex; + justify-content: space-between; + align-items: center; + width: 100%; + gap: 8px; +} diff --git a/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.tsx b/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.tsx new file mode 100644 index 0000000000..12e1430e3c --- /dev/null +++ b/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.tsx @@ -0,0 +1,411 @@ +// The Files-sidebar category manager: a "tune" button opening a modal where you shape the parent categories your files group under. Each category (collapsible, busiest first) has an editable name + icon, a hide toggle, delete, and its member-label chips; add existing team labels to a category, create new categories, reset to the built-in defaults. All device-local (grouping only β€” it never changes the team's label vocabulary); files in no visible category fall back to "Other". + +import { useMemo, useState, useSyncExternalStore } from "react"; +import { useTranslation } from "react-i18next"; +import TuneIcon from "@mui/icons-material/Tune"; +import RestartAltIcon from "@mui/icons-material/RestartAlt"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; +import KeyboardArrowRightIcon from "@mui/icons-material/KeyboardArrowRight"; +import DeleteOutlineIcon from "@mui/icons-material/DeleteOutlined"; +import VisibilityIcon from "@mui/icons-material/Visibility"; +import VisibilityOffIcon from "@mui/icons-material/VisibilityOff"; +import AddIcon from "@mui/icons-material/Add"; +import { TextInput } from "@mantine/core"; +import { Modal } from "@app/ui/Modal"; +import { Button } from "@app/ui/Button"; +import { ActionIcon } from "@app/ui/ActionIcon"; +import { LabelChip } from "@app/ui/LabelChip"; +import { LabelIconPicker } from "@app/components/policies/LabelIconPicker"; +import { useClassificationEnabled } from "@app/hooks/useClassificationEnabled"; +import { useClassificationLabels } from "@app/hooks/useClassificationLabels"; +import { DEFAULT_LABEL_ICON } from "@app/data/labelIcons"; +import { bucketStubsByLabel } from "@app/components/shared/fileSidebarGroupingLogic"; +import { + addCategory, + addLabelToCategory, + deleteCategory, + getSidebarCategories, + removeLabelFromCategory, + renameCategory, + resetSidebarCategories, + setCategoryHidden, + setCategoryIcon, + subscribeSidebarCategories, +} from "@app/services/fileSidebarCategories"; +import type { StirlingFileStub } from "@app/types/fileContext"; +import "@app/components/shared/FileSidebarGroupControls.css"; + +interface FileSidebarGroupControlsProps { + /** The files currently listed, for live per-label counts. */ + stubs: StirlingFileStub[]; +} + +/** New categories start with a neutral folder icon the user can change. */ +const NEW_CATEGORY_ICON = "folder"; + +export function FileSidebarGroupControls({ + stubs, +}: FileSidebarGroupControlsProps) { + const { t } = useTranslation(); + const enabled = useClassificationEnabled(); + const [open, setOpen] = useState(false); + const [query, setQuery] = useState(""); + const categories = useSyncExternalStore( + subscribeSidebarCategories, + getSidebarCategories, + ); + // Only fetch the team label set while the picker is open. + const { teamLabels: labelSet } = useClassificationLabels(open); + + // Bucketed once and reused for both the per-label and per-category counts below. + const byLabel = useMemo(() => bucketStubsByLabel(stubs), [stubs]); + + // Per-label file counts from the same bucketing the sidebar groups use. + const labelCounts = useMemo(() => { + const counts = new Map(); + for (const [key, bucket] of byLabel) counts.set(key, bucket.stubs.length); + return counts; + }, [byLabel]); + + // Files per category (deduped across its labels). + const categoryCounts = useMemo(() => { + const counts = new Map(); + for (const category of categories) { + const ids = new Set(); + for (const key of category.labelKeys) { + for (const stub of byLabel.get(key)?.stubs ?? []) { + ids.add(stub.id as string); + } + } + counts.set(category.id, ids.size); + } + return counts; + }, [byLabel, categories]); + + // Translated display name + icon per label id, plus a nameβ†’id lookup for the + // add-label input (which matches on the visible/canonical text). + const vocab = useMemo(() => { + const byId = new Map(); + const idByName = new Map(); + for (const label of labelSet) { + const display = t(`classification.labels.${label.id}`, label.name); + byId.set(label.id, { display, icon: label.icon }); + idByName.set(display.toLowerCase(), label.id); + idByName.set(label.name.toLowerCase(), label.id); + } + return { byId, idByName }; + }, [labelSet, t]); + + const labelDisplay = (id: string) => vocab.byId.get(id)?.display ?? id; + const labelIcon = (id: string) => + vocab.byId.get(id)?.icon ?? DEFAULT_LABEL_ICON; + + const q = query.trim().toLowerCase(); + const matches = (text: string) => q === "" || text.toLowerCase().includes(q); + + // Busiest categories first (ties keep declaration order). + const sortedCategories = useMemo( + () => + [...categories].sort( + (a, b) => + (categoryCounts.get(b.id) ?? 0) - (categoryCounts.get(a.id) ?? 0), + ), + [categories, categoryCounts], + ); + + // Per-category collapse; on open only the busiest starts expanded. + const [expanded, setExpanded] = useState>(new Set()); + const toggleExpanded = (id: string) => + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + + // Inline rename + add-label drafts, keyed by category id. + const [renaming, setRenaming] = useState(null); + const [renameDraft, setRenameDraft] = useState(""); + const [addDraft, setAddDraft] = useState>({}); + const [newCategory, setNewCategory] = useState(""); + + const openPicker = () => { + setQuery(""); + setRenaming(null); + setNewCategory(""); + setExpanded( + new Set(sortedCategories.length > 0 ? [sortedCategories[0].id] : []), + ); + setOpen(true); + }; + + const commitRename = (id: string) => { + const name = renameDraft.trim(); + if (name) renameCategory(id, name); + setRenaming(null); + }; + + // Add an existing team label to a category (device-local grouping only β€” categories don't change + // the team vocabulary). A name that isn't a known team label is ignored; the input's datalist + // steers callers to real ones. + const addLabel = (categoryId: string) => { + const name = (addDraft[categoryId] ?? "").trim(); + if (!name) return; + const labelId = vocab.idByName.get(name.toLowerCase()); + if (!labelId) return; + addLabelToCategory(categoryId, labelId); + setAddDraft((prev) => ({ ...prev, [categoryId]: "" })); + }; + + const createCategory = () => { + const name = newCategory.trim(); + if (!name) return; + const id = addCategory(name, NEW_CATEGORY_ICON); + setExpanded((prev) => new Set(prev).add(id)); + setNewCategory(""); + }; + + // Classification off (non-AI SaaS tenant) β†’ no "customize groups" affordance, + // matching the flat, ungrouped list. (All hooks above run unconditionally.) + if (!enabled) return null; + + return ( + <> + {/* -external: revealed on section-header hover, like the Browse button. */} + + + + + setOpen(false)} + width="xl" + className="fsg-modal" + title={t("fileSidebar.groupsModal.title", "Sidebar categories")} + subtitle={t( + "fileSidebar.groupsModal.subtitle", + "Group your files into parent categories. Add existing or new labels to a category, rename it, or create your own. Files in none of your categories appear under β€œOther”.", + )} + footer={ +
+ + +
+ } + > +
+ setQuery(e.currentTarget.value)} + placeholder={t("fileSidebar.groupsModal.search", "Search labels…")} + size="sm" + data-autofocus + /> + + {sortedCategories.map((category) => { + const memberKeys = category.labelKeys.filter((key) => + matches(labelDisplay(key)), + ); + // Show the category if its name matches, or any member label matches. + if (!matches(category.name) && memberKeys.length === 0) return null; + const isExpanded = q !== "" || expanded.has(category.id); + const suggestions = [...vocab.byId.keys()] + .filter((id) => !category.labelKeys.includes(id)) + .map((id) => labelDisplay(id)); + return ( +
+
+ toggleExpanded(category.id)} + > + {isExpanded ? ( + + ) : ( + + )} + + setCategoryIcon(category.id, icon)} + ariaLabel={t( + "fileSidebar.groupsModal.categoryIconAria", + "Choose an icon for {{name}}", + { name: category.name }, + )} + /> + {renaming === category.id ? ( + setRenameDraft(e.currentTarget.value)} + onBlur={() => commitRename(category.id)} + onKeyDown={(e) => { + if (e.key === "Enter") commitRename(category.id); + if (e.key === "Escape") setRenaming(null); + }} + /> + ) : ( + + )} + + {categoryCounts.get(category.id) ?? 0} + + + setCategoryHidden(category.id, !category.hidden) + } + > + {category.hidden ? ( + + ) : ( + + )} + + deleteCategory(category.id)} + > + + +
+ + {isExpanded && ( +
+
+ {memberKeys.map((key) => ( + + removeLabelFromCategory(category.id, key) + } + removeAriaLabel={t( + "fileSidebar.groupsModal.removeLabel", + "Remove {{name}}", + { name: labelDisplay(key) }, + )} + /> + ))} +
+
+ + setAddDraft((prev) => ({ + ...prev, + [category.id]: e.target.value, + })) + } + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + addLabel(category.id); + } + }} + /> + + {suggestions.map((name) => ( + + +
+
+ )} +
+ ); + })} + +
+ setNewCategory(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + createCategory(); + } + }} + /> + +
+
+
+ + ); +} diff --git a/frontend/editor/src/saas/components/shared/config/types.ts b/frontend/editor/src/saas/components/shared/config/types.ts index f80bd1ac19..ebaff637d5 100644 --- a/frontend/editor/src/saas/components/shared/config/types.ts +++ b/frontend/editor/src/saas/components/shared/config/types.ts @@ -1,3 +1,4 @@ +import type React from "react"; import { VALID_NAV_KEYS as CORE_NAV_KEYS } from "@core/components/shared/config/types"; // SaaS adds an "overview" account section and an "mcp" integrations tab. All @@ -7,3 +8,21 @@ import { VALID_NAV_KEYS as CORE_NAV_KEYS } from "@core/components/shared/config/ export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview", "mcp"] as const; export type NavKey = (typeof VALID_NAV_KEYS)[number]; + +// Mirrors the core shape over the widened saas NavKey union β€” see the core +// module for why these live in types rather than configNavSections. +export interface ConfigNavItem { + key: NavKey; + label: string; + icon: string; + component: React.ReactNode; + disabled?: boolean; + disabledTooltip?: string; + badge?: string; + badgeColor?: string; +} + +export interface ConfigNavSection { + title: string; + items: ConfigNavItem[]; +} diff --git a/frontend/editor/src/saas/components/shared/fileSidebarGrouping.test.ts b/frontend/editor/src/saas/components/shared/fileSidebarGrouping.test.ts new file mode 100644 index 0000000000..ee55eabb5e --- /dev/null +++ b/frontend/editor/src/saas/components/shared/fileSidebarGrouping.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect } from "vitest"; +import { buildLabelGroups } from "@app/components/shared/fileSidebarGroupingLogic"; +import type { SidebarCategory } from "@app/services/fileSidebarCategories"; +import type { StirlingFileStub } from "@app/types/fileContext"; + +// The grouping only reads id/lastModified/classificationLabels. classificationLabels +// hold label IDS (a file's stored classification), and categories key on ids too. +function stub( + id: string, + labels?: string[], + lastModified = 0, +): StirlingFileStub { + return { id, lastModified, classificationLabels: labels } as StirlingFileStub; +} + +const t = (_key: string, fallback: string) => fallback; + +function cat( + id: string, + name: string, + labelKeys: string[], + hidden = false, +): SidebarCategory { + return { id, name, icon: "folder", labelKeys, hidden }; +} + +const LEGAL = cat("legal", "Legal", ["nda", "contract"]); +const FINANCE = cat("finance", "Financial", ["invoice"]); + +describe("buildLabelGroups", () => { + it("rolls a category's labels into ONE group (deduped)", () => { + const groups = buildLabelGroups( + [ + stub("a", ["nda"]), + stub("b", ["contract"]), + stub("c", ["nda", "contract"]), + ], + [], + t, + [LEGAL], + )!; + const legal = groups.find((g) => g.id === "category:legal")!; + expect(legal.label).toBe("Legal"); + expect(legal.stubs.map((s) => s.id)).toEqual(["a", "b", "c"]); + }); + + it("gives a label not in any category its own group", () => { + const groups = buildLabelGroups( + [stub("a", ["weird-one"])], + [{ id: "weird-one", name: "Weird one" }], + t, + [LEGAL], + )!; + const group = groups.find((g) => g.id === "label:weird-one")!; + expect(group.label).toBe("Weird one"); + expect(group.stubs.map((s) => s.id)).toEqual(["a"]); + }); + + it("moves a hidden category's files into Other", () => { + const groups = buildLabelGroups([stub("a", ["invoice"])], [], t, [ + cat("finance", "Financial", ["invoice"], true), + ])!; + expect(groups.some((g) => g.id === "category:finance")).toBe(false); + expect(groups.at(-1)!.id).toBe("other"); + expect(groups.at(-1)!.stubs.map((s) => s.id)).toEqual(["a"]); + }); + + it("a file with labels in two categories appears in both", () => { + const groups = buildLabelGroups([stub("a", ["nda", "invoice"])], [], t, [ + LEGAL, + FINANCE, + ])!; + expect( + groups.find((g) => g.id === "category:legal")!.stubs.map((s) => s.id), + ).toEqual(["a"]); + expect( + groups.find((g) => g.id === "category:finance")!.stubs.map((s) => s.id), + ).toEqual(["a"]); + expect(groups.some((g) => g.id === "other")).toBe(false); + }); + + it("puts unlabelled files in Other at the bottom", () => { + const groups = buildLabelGroups( + [stub("a", ["invoice"]), stub("b"), stub("c", [])], + [], + t, + [FINANCE], + )!; + const other = groups.at(-1)!; + expect(other.id).toBe("other"); + expect(other.stubs.map((s) => s.id)).toEqual(["b", "c"]); + }); + + it("orders groups: Recent, groups alphabetically, Other last", () => { + const groups = buildLabelGroups( + [ + stub("a", ["invoice"]), + stub("b", ["nda"]), + stub("c", ["zzz"]), + stub("d"), + ], + [], + t, + [LEGAL, FINANCE], + )!; + expect(groups.map((g) => g.id)).toEqual([ + "recent", + "category:finance", + "category:legal", + "label:zzz", + "other", + ]); + }); + + it("returns null for an empty library", () => { + expect(buildLabelGroups([], [], t, [LEGAL])).toBeNull(); + }); +}); diff --git a/frontend/editor/src/saas/components/shared/fileSidebarGrouping.tsx b/frontend/editor/src/saas/components/shared/fileSidebarGrouping.tsx new file mode 100644 index 0000000000..911b44de26 --- /dev/null +++ b/frontend/editor/src/saas/components/shared/fileSidebarGrouping.tsx @@ -0,0 +1,119 @@ +// Classification override of the Files-sidebar grouping seam: Recent, one group per VISIBLE category (device-local, editable β€” default from the built-in label families), a standalone group for any label not yet in a category, then Other for files in none of those. Labels are cached on the stub via a lazy metadata backfill so grouping stays cheap. + +import { + useEffect, + useMemo, + useRef, + useState, + useSyncExternalStore, +} from "react"; +import { useTranslation } from "react-i18next"; +import { useIndexedDB } from "@app/contexts/IndexedDBContext"; +import { useClassificationEnabled } from "@app/hooks/useClassificationEnabled"; +import { useClassificationLabels } from "@app/hooks/useClassificationLabels"; +import { fileStorage } from "@app/services/fileStorage"; +import { readStubClassificationLabels } from "@app/services/fileClassification"; +import { hasInFlightPolicyRuns } from "@app/components/policies/policyRunStore"; +import { + getSidebarCategories, + subscribeSidebarCategories, +} from "@app/services/fileSidebarCategories"; +import { buildLabelGroups } from "@app/components/shared/fileSidebarGroupingLogic"; +import type { FileId } from "@app/types/file"; +import type { StirlingFileStub } from "@app/types/fileContext"; +import type { FileSidebarGroup } from "@core/components/shared/fileSidebarGrouping"; + +export type { FileSidebarGroup }; +// Pure grouping logic lives in a component-free module so tests don't drag in the picker's UI deps. +export { + buildLabelGroups, + bucketStubsByLabel, +} from "@app/components/shared/fileSidebarGroupingLogic"; +// The sidebar's group-picker button + modal (core renders a null stub). +export { FileSidebarGroupControls } from "@app/components/shared/FileSidebarGroupControls"; + +/** Files read per effect pass, so a big library backfills over several ticks. */ +const BACKFILL_BATCH = 3; +/** Recheck delay when the backfill yields to an active policy wave. */ +const BACKFILL_BUSY_RETRY_MS = 4000; + +/** Schedule work for the browser's idle time (or soon after, as a fallback). */ +function scheduleIdle(task: () => void): () => void { + if (typeof requestIdleCallback === "function") { + const handle = requestIdleCallback(task, { timeout: 2000 }); + return () => cancelIdleCallback(handle); + } + const timer = window.setTimeout(task, 200); + return () => window.clearTimeout(timer); +} + +export function useFileSidebarGroups( + stubs: StirlingFileStub[], +): FileSidebarGroup[] | null { + const { t } = useTranslation(); + // Classification off (AI disabled) β†’ no grouping at all: return the flat list + // like core, and don't fetch team labels or backfill from metadata. Gates the + // whole feature so an AI-off SaaS tenant sees no Recent/Other/category chrome. + const enabled = useClassificationEnabled(); + const { teamLabels: labelSet } = useClassificationLabels(enabled); + const { bumpRevision } = useIndexedDB(); + // Attempted reads keyed by id+lastModified: a re-classified file (new version bumps lastModified) is re-read and leaves "Other" on its own, while a truly-unlabelled file keeps a stable key and is read once. + const attempted = useRef>(new Set()); + const attemptKey = (s: StirlingFileStub) => + `${s.id as string}:${s.lastModified ?? 0}`; + // Bumped to re-attempt a backfill pass that yielded to an active policy wave. + const [retryTick, setRetryTick] = useState(0); + + // Fallback for files that arrive with labels already in metadata but no policy delivery (imports/shares): read+cache a few per idle pass, yielding while a policy wave is in flight since those stubs get stamped on delivery anyway. + useEffect(() => { + if (!enabled) return; + const pending = stubs + .filter( + (s) => !s.classificationLabels && !attempted.current.has(attemptKey(s)), + ) + .slice(0, BACKFILL_BATCH); + if (pending.length === 0) return; + let cancelled = false; + let retryTimer: number | undefined; + const cancelIdle = scheduleIdle(() => { + if (cancelled) return; + // Deliveries stamp labels during a wave, so reading now is wasted parsing; recheck after it (a timer self-heals when a wave ends without a stubs change). + if (hasInFlightPolicyRuns()) { + retryTimer = window.setTimeout(() => { + if (!cancelled) setRetryTick((n) => n + 1); + }, BACKFILL_BUSY_RETRY_MS); + return; + } + void (async () => { + let wrote = false; + for (const stub of pending) { + attempted.current.add(attemptKey(stub)); + const labels = await readStubClassificationLabels(stub); + if (cancelled) return; + if (labels) { + const ok = await fileStorage.updateFileMetadata(stub.id as FileId, { + classificationLabels: labels, + }); + if (ok) wrote = true; + } + } + // One revision bump per batch β†’ the sidebar re-reads and re-groups. + if (!cancelled && wrote) bumpRevision(); + })(); + }); + return () => { + cancelled = true; + cancelIdle(); + if (retryTimer !== undefined) window.clearTimeout(retryTimer); + }; + }, [enabled, stubs, bumpRevision, retryTick]); + + const categories = useSyncExternalStore( + subscribeSidebarCategories, + getSidebarCategories, + ); + return useMemo( + () => (enabled ? buildLabelGroups(stubs, labelSet, t, categories) : null), + [enabled, stubs, labelSet, t, categories], + ); +} diff --git a/frontend/editor/src/saas/components/shared/fileSidebarGroupingLogic.ts b/frontend/editor/src/saas/components/shared/fileSidebarGroupingLogic.ts new file mode 100644 index 0000000000..5b61d4a18b --- /dev/null +++ b/frontend/editor/src/saas/components/shared/fileSidebarGroupingLogic.ts @@ -0,0 +1,134 @@ +// Pure grouping logic for the Files sidebar, split from the React seam module so it can be imported (and unit-tested) without pulling in the category-picker component and its heavy UI deps. + +import { DEFAULT_LABEL_ICON } from "@app/data/labelIcons"; +import { accentColor, accentCycleColor } from "@app/utils/accentColors"; +import { + categorizedLabelKeys, + type SidebarCategory, +} from "@app/services/fileSidebarCategories"; +import type { StirlingFileStub } from "@app/types/fileContext"; +import type { FileSidebarGroup } from "@core/components/shared/fileSidebarGrouping"; + +export type { FileSidebarGroup }; + +/** Files shown in the always-expanded "Recent" group. */ +const RECENT_COUNT = 8; + +/** Per label id (a file's stored classification ids) β†’ the stubs carrying it. */ +export function bucketStubsByLabel( + stubs: StirlingFileStub[], +): Map { + const byLabel = new Map(); + for (const stub of stubs) { + for (const labelId of stub.classificationLabels ?? []) { + const bucket = byLabel.get(labelId); + if (bucket) bucket.stubs.push(stub); + else byLabel.set(labelId, { stubs: [stub] }); + } + } + return byLabel; +} + +/** + * Pure grouping: Recent (top {@link RECENT_COUNT} by lastModified), then visible groups sorted + * alphabetically, then Other (files in no visible group) last. Visible groups are the non-hidden + * {@link SidebarCategory} entries with β‰₯1 file (a file lands in every category it has a label in), + * plus a standalone group for any label on a file that isn't in a category yet. + */ +export function buildLabelGroups( + stubs: StirlingFileStub[], + labelSet: readonly { id: string; name: string; icon?: string }[], + t: (key: string, fallback: string) => string, + categories: SidebarCategory[], +): FileSidebarGroup[] | null { + if (stubs.length === 0) return null; + + // Display name + icon per label id from the effective set; ids no longer in the + // set still group, resolving to the id text and the default icon. + const nameById = new Map(); + const iconById = new Map(); + for (const label of labelSet) { + nameById.set(label.id, label.name); + iconById.set(label.id, label.icon); + } + const labelName = (id: string) => + t(`classification.labels.${id}`, nameById.get(id) ?? id); + + const recent = [...stubs] + .sort((a, b) => (b.lastModified ?? 0) - (a.lastModified ?? 0)) + .slice(0, RECENT_COUNT); + + const byLabel = bucketStubsByLabel(stubs); + const inACategory = categorizedLabelKeys(categories); + + const visible: Omit[] = []; + + // One group per visible category: every file carrying any of its labels, in the input's order + // (deduped β€” a file with two of the category's labels appears once). + for (const category of categories) { + if (category.hidden) continue; + const ids = new Set(category.labelKeys); + const members = stubs.filter((stub) => + (stub.classificationLabels ?? []).some((labelId) => ids.has(labelId)), + ); + if (members.length === 0) continue; + visible.push({ + id: `category:${category.id}`, + label: category.name, + icon: category.icon, + stubs: members, + defaultExpanded: false, + }); + } + + // A label on a file but not in any category still gets its own group, so nothing is stranded + // until the user files it into a category. + for (const [labelId, bucket] of byLabel) { + if (inACategory.has(labelId)) continue; + visible.push({ + id: `label:${labelId}`, + label: labelName(labelId), + icon: iconById.get(labelId) ?? DEFAULT_LABEL_ICON, + stubs: bucket.stubs, + defaultExpanded: false, + }); + } + + visible.sort((a, b) => + a.label.localeCompare(b.label, undefined, { sensitivity: "base" }), + ); + + // Other = files in no visible group: unlabelled, or labelled only under hidden categories. + const covered = new Set(); + for (const group of visible) { + for (const stub of group.stubs) covered.add(stub.id as string); + } + const other = stubs.filter((stub) => !covered.has(stub.id as string)); + + const groups: FileSidebarGroup[] = [ + { + id: "recent", + label: t("fileSidebar.recent", "Recent"), + icon: "history", + stubs: recent, + defaultExpanded: true, + }, + ...visible.map((group, index) => ({ + ...group, + // Theme-adaptive accent cycled in display order, so each icon reads at a glance. + color: accentCycleColor(index), + })), + ]; + if (other.length > 0) { + groups.push({ + id: "other", + label: t("fileSidebar.other", "Other"), + icon: DEFAULT_LABEL_ICON, + // Neutral grey for the "Other" (unlabelled) group. + color: accentColor("gray"), + stubs: other, + defaultExpanded: false, + }); + } + return groups; +} diff --git a/frontend/editor/src/saas/hooks/useClassificationEnabled.ts b/frontend/editor/src/saas/hooks/useClassificationEnabled.ts new file mode 100644 index 0000000000..a776f77986 --- /dev/null +++ b/frontend/editor/src/saas/hooks/useClassificationEnabled.ts @@ -0,0 +1,11 @@ +// SaaS override of the classification-enabled seam: classification is available +// exactly when the AI engine is on for this tenant. Off β†’ the sidebar grouping, +// group-picker, per-file label chips and the file-details Classification section +// all stay hidden, so an AI-disabled SaaS tenant sees the plain flat file list +// with no hint the feature exists. + +import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled"; + +export function useClassificationEnabled(): boolean { + return useAiEngineEnabled(); +} diff --git a/frontend/editor/src/saas/portal/usersCapabilities.ts b/frontend/editor/src/saas/portal/usersCapabilities.ts new file mode 100644 index 0000000000..e24ee29745 --- /dev/null +++ b/frontend/editor/src/saas/portal/usersCapabilities.ts @@ -0,0 +1,25 @@ +import type { UsersCapabilities } from "@portal/api/usersCapabilities"; + +/** + * SaaS build: team-leader scope. No org-wide roster, no create-team, and no + * password/role administration (Supabase-authed users); leaders invite, rename, + * remove members, and manage seats. Mirrors what SaasTeamController exposes. + */ +export const usersCapabilities: UsersCapabilities = { + orgGroup: false, + changeRole: false, + adminRole: false, + createTeam: false, + deleteTeam: false, + renameTeam: true, + emailInvite: true, + directCreate: false, + resetPassword: false, + unlock: false, + resetMfa: false, + suspend: false, + moveTeam: false, + seats: true, + manageGrants: false, + removeScope: "team", +}; diff --git a/frontend/editor/vite.config.ts b/frontend/editor/vite.config.ts index 17c0c09282..059f98332e 100644 --- a/frontend/editor/vite.config.ts +++ b/frontend/editor/vite.config.ts @@ -6,7 +6,7 @@ import { constants, brotliCompress, gzip } from "node:zlib"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { defineConfig, loadEnv } from "vite"; -import type { PluginOption } from "vite"; +import type { Connect, PluginOption } from "vite"; import tsconfigPaths from "vite-tsconfig-paths"; import { viteStaticCopy } from "vite-plugin-static-copy"; @@ -129,6 +129,38 @@ function prerenderOgPlugin(): PluginOption { }; } +/** + * When the app is served under a subpath (RUN_SUBPATH β†’ base like "/app/"), Vite + * serves index.html at "/app/" and redirects "/" β†’ the base, but a bare "/app" + * (no trailing slash) 404s. This middleware redirects "/app" β†’ "/app/" so either + * form loads the app in dev and `vite preview`. Query strings are preserved. + */ +function subpathBareRedirectPlugin(subpath: string): PluginOption { + const bare = `/${subpath}`; + const withSlash = `${bare}/`; + const redirect: Connect.NextHandleFunction = (req, res, next) => { + const url = req.url ?? ""; + const q = url.indexOf("?"); + const pathname = q === -1 ? url : url.slice(0, q); + if (pathname === bare) { + res.statusCode = 301; + res.setHeader("Location", withSlash + (q === -1 ? "" : url.slice(q))); + res.end(); + return; + } + next(); + }; + return { + name: "subpath-bare-redirect", + configureServer(server) { + server.middlewares.use(redirect); + }, + configurePreviewServer(server) { + server.middlewares.use(redirect); + }, + }; +} + // NOTE: cloud/ is a SHARED layer, not a runnable build flavor β€” it's compiled // into the saas and desktop builds. It has no entry here and no vite tsconfig; // it is only typechecked standalone via editor/src/cloud/tsconfig.json @@ -178,6 +210,9 @@ export default defineConfig(async ({ mode }) => { const tsconfigProject = TSCONFIG_MAP[effectiveMode]; + // Subpath the app is served under (base becomes "//"). Empty = root. + const runSubpath = (env.RUN_SUBPATH || "").replace(/^\/+|\/+$/g, ""); + // Backend proxy target: default localhost:8080. Override via BACKEND_URL env var // so the top-level dev launcher can wire a dynamically-assigned backend port. const backendUrl = process.env.BACKEND_URL || "http://localhost:8080"; @@ -217,6 +252,7 @@ export default defineConfig(async ({ mode }) => { return { plugins: [ react(), + ...(runSubpath ? [subpathBareRedirectPlugin(runSubpath)] : []), tsconfigPaths({ projects: [tsconfigProject], }), @@ -330,8 +366,8 @@ export default defineConfig(async ({ mode }) => { // an absolute base so deep-route asset paths resolve to /assets/... // Trailing slash required: it becomes ``, and browsers resolve // relative URLs (manifest.json, favicon) against the base's *directory*. - base: env.RUN_SUBPATH - ? `/${env.RUN_SUBPATH}/` + base: runSubpath + ? `/${runSubpath}/` : process.env.VITE_BUILD_FOR_PREVIEW === "1" ? "/" : "./", diff --git a/frontend/eslint.config.mjs b/frontend/eslint.config.mjs index d954ca4006..5c3d20b94a 100644 --- a/frontend/eslint.config.mjs +++ b/frontend/eslint.config.mjs @@ -240,6 +240,17 @@ export default defineConfig( "no-restricted-syntax": ["error", ...mantineComponentImportRestrictions], }, }, + // TEMPORARY: the portal user-management / integrations surface predates the + // button consolidation and uses bespoke CSS-styled raw