diff --git a/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java b/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java index 7b74111870..aa521bf942 100644 --- a/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java +++ b/app/common/src/main/java/stirling/software/common/service/PdfMetadataService.java @@ -7,6 +7,7 @@ import java.time.format.DateTimeFormatter; import java.util.Calendar; import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDDocumentInformation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -17,6 +18,9 @@ import stirling.software.common.model.PdfMetadata; @Service public class PdfMetadataService { + /** ({@code {labels}}). Written by the classify-and-label tool. */ + public static final String CLASSIFICATION_KEY = "StirlingPDFClassification"; + private final ApplicationProperties applicationProperties; private final String stirlingPDFLabel; private final UserServiceInterface userService; @@ -177,4 +181,14 @@ public class PdfMetadataService { } pdf.getDocumentInformation().setAuthor(author); } + + /** + * Write the document classifier's JSON result into the custom Info-dictionary field {@link + * #CLASSIFICATION_KEY}, leaving all other metadata untouched. + */ + public void setClassificationMetadata(PDDocument pdf, String classificationJson) { + PDDocumentInformation info = pdf.getDocumentInformation(); + info.setCustomMetadataValue(CLASSIFICATION_KEY, classificationJson); + pdf.setDocumentInformation(info); + } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java index f61a1ce8fe..a66b168163 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/GetInfoOnPDF.java @@ -305,6 +305,23 @@ public class GetInfoOnPDF { } } + /** + * Info-dictionary keys exposed above via typed getters; any other key in the dictionary is + * surfaced as custom metadata (e.g. the classification policy's StirlingPDFClassification + * entry). + */ + private static final java.util.Set STANDARD_INFO_KEYS = + java.util.Set.of( + "Title", + "Author", + "Subject", + "Keywords", + "Producer", + "Creator", + "CreationDate", + "ModDate", + "Trapped"); + private static ObjectNode extractMetadata(PDDocument document) { ObjectNode metadata = objectMapper.createObjectNode(); @@ -335,6 +352,18 @@ public class GetInfoOnPDF { if (modificationDate != null) { metadata.put("ModificationDate", modificationDate); } + + // Surface custom Info-dictionary entries (anything beyond the + // standard fields above) — e.g. StirlingPDFClassification + for (String key : info.getMetadataKeys()) { + if (STANDARD_INFO_KEYS.contains(key)) { + continue; + } + String value = info.getCustomMetadataValue(key); + if (value != null && !value.isBlank()) { + metadata.put(key, value); + } + } } } catch (Exception e) { log.error("Error extracting metadata: {}", e.getMessage()); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationLabelsController.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationLabelsController.java new file mode 100644 index 0000000000..6dcd19e724 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationLabelsController.java @@ -0,0 +1,136 @@ +package stirling.software.proprietary.classification; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.UserServiceInterface; +import stirling.software.proprietary.classification.model.ClassificationLabels; +import stirling.software.proprietary.classification.model.LabelsValidator; +import stirling.software.proprietary.classification.store.ClassificationLabelStore; +import stirling.software.proprietary.classification.store.TeamLabelsEntity; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; + +/** + * Read/write the team's classification label set — the flat vocabulary the document classifier runs + * against. Shared and team-scoped exactly like policies: every user reads their own team's labels, + * and only a user who may edit policies (a team leader on SaaS, the global admin self-hosted; see + * {@link PolicyManagementAuthority}) may change it — gated only when login is enabled, since + * single-user deployments trust the local operator. A team with no stored labels reads as {@code + * 204}; that team has no vocabulary, so its documents are not classified (there is no built-in + * default on the backend or the engine — the label data lives only in the frontend). + */ +@RestController +@RequestMapping("/api/v1/classification/labels") +@Hidden +@RequiredArgsConstructor +@Tag(name = "Classification", description = "Team-scoped document-classification labels") +@ConditionalOnBooleanProperty(name = "policies.enabled") +public class ClassificationLabelsController { + + private final ClassificationLabelStore labelStore; + private final PolicyManagementAuthority policyManagementAuthority; + private final ApplicationProperties applicationProperties; + private final UserServiceInterface userService; + + @GetMapping + @Operation( + summary = "Get the team's classification labels", + description = + "Returns the caller's team label set, or 204 when the team has none (its" + + " documents are then not classified).") + public ResponseEntity getTeamLabels() { + return labelStore + .findByTeam(currentTeamId()) + .map(ResponseEntity::ok) + .orElseGet(() -> ResponseEntity.noContent().build()); + } + + @PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE) + @Operation( + summary = "Save the team's classification labels", + description = + "Validates and stores the label set for the caller's team, shared by everyone" + + " on the team. Requires the policy-editor role for the team.") + public ResponseEntity saveTeamLabels( + @RequestBody ClassificationLabels labels) { + requireEditingAllowed(); + validate(labels); + ClassificationLabels saved = labelStore.save(currentTeamId(), labels, currentUsername()); + return ResponseEntity.ok(saved); + } + + @DeleteMapping + @Operation( + summary = "Reset the team's classification labels", + description = + "Removes the team's stored label set; its documents are then not classified" + + " until labels are saved again. Requires the policy-editor role for the" + + " team.") + public ResponseEntity resetTeamLabels() { + requireEditingAllowed(); + labelStore.deleteByTeam(currentTeamId()); + return ResponseEntity.noContent().build(); + } + + private static void validate(ClassificationLabels labels) { + try { + LabelsValidator.validate(labels); + } catch (IllegalArgumentException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + } + + /** + * Editing the team labels requires the editor role for the caller's team — the same gate + * policies use (team leader on SaaS, global admin self-hosted). Single-user deployments (login + * disabled) have no such role, so they trust the local operator. + */ + private void requireEditingAllowed() { + if (!applicationProperties.getSecurity().isEnableLogin()) { + return; + } + if (!policyManagementAuthority.canEditPolicies()) { + throw new ResponseStatusException( + HttpStatus.FORBIDDEN, + "The team classification labels may only be changed by a team leader"); + } + } + + /** + * The caller's team key. With login disabled the single operator owns the {@link + * TeamLabelsEntity#NO_TEAM} sentinel row; with login enabled a caller with no resolvable team + * is an error rather than being dropped into the shared sentinel bucket (which would let + * unteamed users read and overwrite each other's "team" labels). + */ + private Long currentTeamId() { + Long teamId = policyManagementAuthority.currentUserTeamId(); + if (teamId != null) { + return teamId; + } + if (!applicationProperties.getSecurity().isEnableLogin()) { + return TeamLabelsEntity.NO_TEAM; + } + throw new ResponseStatusException( + HttpStatus.UNAUTHORIZED, "Could not resolve the current user's team"); + } + + private String currentUsername() { + return userService == null ? null : userService.getCurrentUsername(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabel.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabel.java new file mode 100644 index 0000000000..6601cba7e5 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabel.java @@ -0,0 +1,9 @@ +package stirling.software.proprietary.classification.model; + +/** + * One entry in the classification vocabulary. {@code id} is the label's stable identity (a slug, + * unique within a set): it is what the engine returns and what is stored on the document. {@code + * name} is the human display text the classifier model reasons over. {@code icon} is an optional + * presentational key (a Material Symbols name shown in the file sidebar); the engine never sees it. + */ +public record ClassificationLabel(String id, String name, String icon) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabels.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabels.java new file mode 100644 index 0000000000..ea7a1269f0 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/ClassificationLabels.java @@ -0,0 +1,16 @@ +package stirling.software.proprietary.classification.model; + +import java.util.List; + +/** + * A flat multi-label classification vocabulary — the set of labels a document may be assigned. + * Stored per team (admin-edited, shared by everyone on the team); the classifier runs against these + * label names. A team with no stored set has no vocabulary, so its documents are not classified — + * neither the backend nor the engine holds a default of its own. + */ +public record ClassificationLabels(List labels) { + + public ClassificationLabels { + labels = labels == null ? List.of() : List.copyOf(labels); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/LabelsValidator.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/LabelsValidator.java new file mode 100644 index 0000000000..722e8b417b --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/model/LabelsValidator.java @@ -0,0 +1,71 @@ +package stirling.software.proprietary.classification.model; + +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Structural validation for a user- or admin-supplied label set, run before it is stored so a + * malformed vocabulary can never reach the classifier. Mirrors the invariants the engine relies on: + * non-blank ids and names, each unique within the set (ids exactly, names case-insensitively). + */ +public final class LabelsValidator { + + private LabelsValidator() {} + + // Generous upper bounds so a legitimate label set is never blocked, but a single team or user + // can't store an unbounded blob that would bloat the row, balloon the classifier prompt, or + // exhaust memory on deserialize. + static final int MAX_LABELS = 500; + static final int MAX_TEXT_LENGTH = 128; + + // Icon is a Material Symbols key (lowercase, digits, hyphens). Enforce the SHAPE server-side — + // the exact allowlist lives in the frontend — so a client bypassing the UI can't store + // arbitrary + // text that would render as garbage (or worse) in every teammate's sidebar. + private static final Pattern ICON_KEY = Pattern.compile("^[a-z0-9-]+$"); + + /** + * @throws IllegalArgumentException with a human-readable message when the label set is invalid. + */ + public static void validate(ClassificationLabels labels) { + if (labels == null || labels.labels() == null) { + throw new IllegalArgumentException("Labels are required"); + } + if (labels.labels().size() > MAX_LABELS) { + throw new IllegalArgumentException("Too many labels (max " + MAX_LABELS + ")"); + } + Set ids = new HashSet<>(); + Set names = new HashSet<>(); + for (ClassificationLabel label : labels.labels()) { + requireText(label.id(), "Label id"); + requireText(label.name(), "Label name"); + if (label.icon() != null && !label.icon().isEmpty()) { + if (label.icon().length() > MAX_TEXT_LENGTH) { + throw new IllegalArgumentException( + "Label icon is too long (max " + MAX_TEXT_LENGTH + " characters)"); + } + if (!ICON_KEY.matcher(label.icon()).matches()) { + throw new IllegalArgumentException("Invalid label icon: " + label.icon()); + } + } + if (!ids.add(label.id().trim())) { + throw new IllegalArgumentException("Duplicate label id: " + label.id()); + } + if (!names.add(label.name().trim().toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException("Duplicate label name: " + label.name()); + } + } + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + if (value.trim().length() > MAX_TEXT_LENGTH) { + throw new IllegalArgumentException( + field + " is too long (max " + MAX_TEXT_LENGTH + " characters)"); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/ClassificationLabelStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/ClassificationLabelStore.java new file mode 100644 index 0000000000..a604854c60 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/ClassificationLabelStore.java @@ -0,0 +1,22 @@ +package stirling.software.proprietary.classification.store; + +import java.util.Optional; + +import stirling.software.proprietary.classification.model.ClassificationLabels; + +/** + * Stores one {@link ClassificationLabels} set per team. A {@code null} teamId addresses the + * unteamed set (login disabled / no resolvable team), mirroring how the policy store treats a null + * team. + */ +public interface ClassificationLabelStore { + + /** The team's stored labels, or empty when it has none (callers then skip classification). */ + Optional findByTeam(Long teamId); + + /** Create or replace the team's labels. Returns the stored value. */ + ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy); + + /** Remove the team's labels (reset to default). Returns whether a set existed. */ + boolean deleteByTeam(Long teamId); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/InProcessClassificationLabelStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/InProcessClassificationLabelStore.java new file mode 100644 index 0000000000..6846902eab --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/InProcessClassificationLabelStore.java @@ -0,0 +1,36 @@ +package stirling.software.proprietary.classification.store; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import stirling.software.proprietary.classification.model.ClassificationLabels; + +/** + * In-memory {@link ClassificationLabelStore} for tests and any future no-database mode. {@link + * JpaClassificationLabelStore} is the runtime bean. + */ +public class InProcessClassificationLabelStore implements ClassificationLabelStore { + + private final Map byTeam = new ConcurrentHashMap<>(); + + @Override + public Optional findByTeam(Long teamId) { + return Optional.ofNullable(byTeam.get(key(teamId))); + } + + @Override + public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) { + byTeam.put(key(teamId), labels); + return labels; + } + + @Override + public boolean deleteByTeam(Long teamId) { + return byTeam.remove(key(teamId)) != null; + } + + private static long key(Long teamId) { + return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/JpaClassificationLabelStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/JpaClassificationLabelStore.java new file mode 100644 index 0000000000..724e335fc4 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/JpaClassificationLabelStore.java @@ -0,0 +1,76 @@ +package stirling.software.proprietary.classification.store; + +import java.time.Instant; +import java.util.Optional; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.classification.model.ClassificationLabels; + +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; + +/** + * Durable {@link ClassificationLabelStore} backed by JPA; the runtime store. Gated on {@code + * policies.enabled} — stored labels only matter when the Classification policy can run — so it + * shares the policy subsystem's on/off switch. Each label set is persisted as JSON via {@link + * TeamLabelsEntity}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnBooleanProperty(name = "policies.enabled") +public class JpaClassificationLabelStore implements ClassificationLabelStore { + + private final TeamLabelsRepository teamRepository; + private final ObjectMapper objectMapper; + + @Override + public Optional findByTeam(Long teamId) { + return teamRepository + .findById(key(teamId)) + .flatMap(entity -> parse(entity.getLabelsJson(), "team " + teamId)); + } + + @Override + public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) { + TeamLabelsEntity entity = new TeamLabelsEntity(); + entity.setTeamId(key(teamId)); + entity.setLabelsJson(objectMapper.writeValueAsString(labels)); + entity.setUpdatedAt(Instant.now()); + entity.setUpdatedBy(updatedBy); + teamRepository.save(entity); + return labels; + } + + @Override + public boolean deleteByTeam(Long teamId) { + long id = key(teamId); + if (!teamRepository.existsById(id)) { + return false; + } + teamRepository.deleteById(id); + return true; + } + + private Optional parse(String json, String owner) { + try { + return Optional.of(objectMapper.readValue(json, ClassificationLabels.class)); + } catch (JacksonException e) { + // A stored label set that no longer parses (corruption / manual DB edit) must not break + // classification: drop it so the caller treats the team as having no labels (and skips + // classification) rather than surfacing a 500 on every upload. + log.warn("Discarding unparseable stored labels for {}: {}", owner, e.getMessage()); + return Optional.empty(); + } + } + + /** Map the nullable team id onto the entity's non-null key (sentinel for the unteamed case). */ + private static long key(Long teamId) { + return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsEntity.java new file mode 100644 index 0000000000..25f2e5b259 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsEntity.java @@ -0,0 +1,47 @@ +package stirling.software.proprietary.classification.store; + +import java.io.Serializable; +import java.time.Instant; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * JPA row for a team's classification labels — one row per team. The label set lives as JSON in + * {@code labelsJson} (authoritative on read). {@code teamId} is the natural key; the sentinel + * {@link #NO_TEAM} stands in for the unteamed (login-disabled / self-hosted single-team) case, + * since a primary key can't be null (policies store a nullable {@code team_id}, but this table is + * keyed one-per-team). Kept decoupled from the security entities — {@code teamId} is a plain value, + * not a foreign key — so classification can be enabled or disabled without touching them. + */ +@Entity +@Table(name = "classification_labels") +@NoArgsConstructor +@Getter +@Setter +public class TeamLabelsEntity implements Serializable { + + private static final long serialVersionUID = 1L; + + /** Sentinel key for the unteamed label set (login disabled / no resolvable team). */ + public static final long NO_TEAM = 0L; + + @Id + @Column(name = "team_id") + private long teamId; + + @Column(name = "labels_json", columnDefinition = "text") + private String labelsJson; + + @Column(name = "updated_at") + private Instant updatedAt; + + @Column(name = "updated_by") + private String updatedBy; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsRepository.java new file mode 100644 index 0000000000..7fc9589c39 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/store/TeamLabelsRepository.java @@ -0,0 +1,7 @@ +package stirling.software.proprietary.classification.store; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface TeamLabelsRepository extends JpaRepository {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java new file mode 100644 index 0000000000..629a377cf3 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java @@ -0,0 +1,218 @@ +package stirling.software.proprietary.controller.api; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import io.github.pixee.security.Filenames; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.common.service.UserServiceInterface; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.WebResponseUtils; +import stirling.software.proprietary.classification.model.ClassificationLabel; +import stirling.software.proprietary.classification.store.ClassificationLabelStore; +import stirling.software.proprietary.model.api.ai.AiPageText; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; +import stirling.software.proprietary.service.AiEngineClient; +import stirling.software.proprietary.service.PdfContentExtractor; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ObjectNode; + +/** + * Dispatchable tool that classifies a PDF and writes the result into its metadata. + * + *

Runs as a Classification-policy pipeline step: it reads a bounded page window, asks the AI + * engine to classify the document against the caller's team label set, and stores the engine's JSON + * answer — minus the transport-only {@code outcome} field — in the custom Info-dictionary key + * {@link PdfMetadataService#CLASSIFICATION_KEY}. Returns the labelled PDF. Not intended for direct + * client use. + */ +@Slf4j +@Hidden +@RestController +@RequestMapping("/api/v1/ai/tools") +@Tag(name = "AI Tools", description = "Dispatchable AI-backed tools.") +public class ClassifyLabelController { + + /** Pages read from each end of the document — mirrors the engine's window. */ + private static final int WINDOW_PAGES = 2; + + private static final String CLASSIFY_ENDPOINT = "/api/v1/documents/classify"; + + private final CustomPDFDocumentFactory pdfDocumentFactory; + private final TempFileManager tempFileManager; + private final PdfContentExtractor pdfContentExtractor; + private final PdfMetadataService pdfMetadataService; + private final AiEngineClient aiEngineClient; + private final ObjectMapper objectMapper; + private final UserServiceInterface userService; + + /** + * Present only when the policy subsystem is enabled ({@code policies.enabled}); the store and + * team authority are gated on it. Null otherwise, in which case there are no team labels to + * classify against and the document is passed through unlabelled. + */ + private final ClassificationLabelStore labelStore; + + private final PolicyManagementAuthority policyManagementAuthority; + + public ClassifyLabelController( + CustomPDFDocumentFactory pdfDocumentFactory, + TempFileManager tempFileManager, + PdfContentExtractor pdfContentExtractor, + PdfMetadataService pdfMetadataService, + AiEngineClient aiEngineClient, + ObjectMapper objectMapper, + @Autowired(required = false) UserServiceInterface userService, + @Autowired(required = false) ClassificationLabelStore labelStore, + @Autowired(required = false) PolicyManagementAuthority policyManagementAuthority) { + this.pdfDocumentFactory = pdfDocumentFactory; + this.tempFileManager = tempFileManager; + this.pdfContentExtractor = pdfContentExtractor; + this.pdfMetadataService = pdfMetadataService; + this.aiEngineClient = aiEngineClient; + this.objectMapper = objectMapper; + this.userService = userService; + this.labelStore = labelStore; + this.policyManagementAuthority = policyManagementAuthority; + } + + @PostMapping(value = "/classify-and-label", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Classify a PDF and label its metadata", + description = + "Reads the first two and last two pages, classifies the document via the AI" + + " engine, and stores the result in the StirlingPDFClassification" + + " metadata field. Dispatched by the Classification policy; not" + + " intended for direct client use.") + public ResponseEntity classifyAndLabel( + @RequestParam("fileInput") MultipartFile fileInput) throws IOException { + try (PDDocument document = pdfDocumentFactory.load(fileInput, true)) { + String fileName = safeFileName(fileInput.getOriginalFilename()); + + List allowed = resolveAllowedLabels(); + if (allowed.isEmpty()) { + // No vocabulary to classify against (the team stored no labels): pass the file + // through unlabelled rather than ask the engine to classify against nothing. + log.debug("[classify-and-label] {} has no team labels; skipping", fileName); + return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager); + } + + List pages = extractWindow(document); + String requestBody = + objectMapper.writeValueAsString( + new ClassifyEngineRequest(fileName, pages, allowed)); + + String userId = userService != null ? userService.getCurrentUsername() : null; + String responseJson = aiEngineClient.post(CLASSIFY_ENDPOINT, requestBody, userId); + + pdfMetadataService.setClassificationMetadata(document, toMetadataValue(responseJson)); + log.debug("[classify-and-label] labelled {} ({} window pages)", fileName, pages.size()); + + return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager); + } + } + + private List extractWindow(PDDocument document) throws IOException { + List pages = new ArrayList<>(); + for (int pageNumber : windowPageNumbers(document.getNumberOfPages(), WINDOW_PAGES)) { + String text = pdfContentExtractor.extractPageTextRaw(document, pageNumber); + if (text != null && !text.isBlank()) { + pages.add(new AiPageText(pageNumber, text)); + } + } + return pages; + } + + /** First and last {@code window} page numbers (1-based), de-duplicated and in order. */ + static List windowPageNumbers(int pageCount, int window) { + Set numbers = new LinkedHashSet<>(); + for (int page = 1; page <= Math.min(window, pageCount); page++) { + numbers.add(page); + } + for (int page = Math.max(1, pageCount - window + 1); page <= pageCount; page++) { + numbers.add(page); + } + return new ArrayList<>(numbers); + } + + /** Drop the transport-only {@code outcome} discriminator; keep the rest verbatim. */ + private String toMetadataValue(String engineResponseJson) { + JsonNode node = objectMapper.readTree(engineResponseJson); + if (node instanceof ObjectNode object) { + object.remove("outcome"); + } + return objectMapper.writeValueAsString(node); + } + + private static String safeFileName(String originalFilename) { + String name = Filenames.toSimpleFileName(originalFilename); + return (name == null || name.isBlank()) ? "classified.pdf" : name; + } + + /** + * The allowed labels for the caller's team as {@code {id, name}} pairs, de-duplicated by id. + * The engine shows the model the names and returns the ids (icons are presentational and never + * sent). Returns an empty list — the caller then skips classification — when the policy + * subsystem is disabled (no store) or the team has no stored labels. The engine holds no + * default vocabulary of its own, so a team's stored labels are the only source. + */ + private List resolveAllowedLabels() { + if (labelStore == null) { + return List.of(); + } + Long teamId = + policyManagementAuthority == null + ? null + : policyManagementAuthority.currentUserTeamId(); + + Map byId = new LinkedHashMap<>(); + labelStore.findByTeam(teamId).ifPresent(labels -> collectLabels(labels.labels(), byId)); + + return List.copyOf(byId.values()); + } + + private static void collectLabels( + List labels, Map into) { + for (ClassificationLabel label : labels) { + if (label.id() == null + || label.id().isBlank() + || label.name() == null + || label.name().isBlank()) { + continue; + } + into.putIfAbsent(label.id(), new EngineLabel(label.id(), label.name())); + } + } + + /** One allowed label sent to the engine: stable id + the name the model reasons over. */ + private record EngineLabel(String id, String name) {} + + /** Request body for the engine's {@code /api/v1/documents/classify} endpoint. */ + private record ClassifyEngineRequest( + String fileName, List pages, List labels) {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java index 57207d2bc9..c80615f03e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResultFile.java @@ -21,4 +21,11 @@ public class AiWorkflowResultFile { @Schema(description = "MIME type of the file", example = "application/pdf") private String contentType; + + @Schema( + description = + "Index into the request's fileInputs that this output was derived from, or null" + + " when it has no single source (e.g. a merge, or a generated file)." + + " Lets the client replace that input in place as a new version.") + private Integer sourceIndex; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 64e6aa0523..3d076bb527 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestPart; @@ -213,6 +214,20 @@ public class PolicyController { return ResponseEntity.ok(saved); } + @PutMapping("/order") + @Operation( + summary = "Set the team's policy run order", + description = + "Persists the team-wide order policies run in, from the given ordered list of" + + " policy ids (position → order). The per-trigger order shown in the UI" + + " is this one sequence filtered by trigger. Team-leader/admin only;" + + " ids outside the caller's team are ignored.") + public ResponseEntity reorderPolicies(@RequestBody List orderedPolicyIds) { + requirePolicyEditingAllowed(); + policyStore.reorder(policyAccessGuard.teamForNewPolicy(), orderedPolicyIds); + return ResponseEntity.noContent().build(); + } + /** * Every {@code sourceId} a policy references must resolve to a source in the caller's team, so * a client can neither reference a non-existent source nor reach across teams to use another diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java index 8bfb78410b..2d5552517c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java @@ -8,7 +8,11 @@ import tools.jackson.databind.JsonNode; /** * Result of a {@link PolicyExecutor} run. {@code files} are final temp files (not yet stored). - * {@code report}/{@code reportTool} carry the last step's structured report and its operation, or - * null if no step produced one. + * {@code origins} is parallel to {@code files}: each entry is the index into the original pipeline + * inputs that the output traces back to, or {@code null} when it has no single source (e.g. a merge + * combining several inputs, or a generated file). Callers use it to map an output back onto the + * file it came from. {@code report}/{@code reportTool} carry the last step's structured report and + * its operation, or null if no step produced one. */ -public record PolicyExecutionResult(List files, JsonNode report, String reportTool) {} +public record PolicyExecutionResult( + List files, List origins, JsonNode report, String reportTool) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java index fe88b40391..d6202b520e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java @@ -58,6 +58,11 @@ public class PolicyExecutor { // payload the tool surfaced alongside or instead of a file. private record ToolResult(List files, JsonNode report) {} + // A step's output files paired with each file's origin (the index into the original pipeline + // inputs it traces back to, or null when it has no single source). Origins compose across steps + // so the final result can be mapped back onto the files that entered the pipeline. + private record StepOutput(List files, List origins, JsonNode report) {} + /** * Run every step in order, feeding each step's output into the next. Supporting files in {@code * inputs} bind to named file fields and never enter the document stream. @@ -75,6 +80,12 @@ public class PolicyExecutor { List currentFiles = inputs.primary(); Map> supportingFiles = inputs.supportingFiles(); + // Seed each input with its own index as origin; steps carry these through so the final + // outputs can be traced back to the files that entered the pipeline. + List currentOrigins = new ArrayList<>(); + for (int k = 0; k < currentFiles.size(); k++) { + currentOrigins.add(k); + } // Last non-null report wins: the terminal step defines the output. JsonNode lastReport = null; String lastReportTool = null; @@ -87,8 +98,10 @@ public class PolicyExecutor { "Pipeline step " + (i + 1) + " has no operation"); } listener.onStepStart(i + 1, steps.size(), operation); - ToolResult stepResult = executeStep(step, currentFiles, supportingFiles); + StepOutput stepResult = + executeStep(step, currentFiles, currentOrigins, supportingFiles); currentFiles = stepResult.files(); + currentOrigins = stepResult.origins(); if (stepResult.report() != null) { lastReport = stepResult.report(); lastReportTool = operation; @@ -96,7 +109,7 @@ public class PolicyExecutor { listener.onStepComplete(i + 1, steps.size(), operation); } - return new PolicyExecutionResult(currentFiles, lastReport, lastReportTool); + return new PolicyExecutionResult(currentFiles, currentOrigins, lastReport, lastReportTool); } /** @@ -104,32 +117,50 @@ public class PolicyExecutor { * responses are unpacked so each inner file is its own result (e.g. split). For per-file * dispatch the first non-null report wins. */ - private ToolResult executeStep( + private StepOutput executeStep( PipelineStep step, List inputFiles, + List inputOrigins, Map> supportingFiles) throws IOException { requireAcceptedTypes(step.operation(), inputFiles); List files = new ArrayList<>(); + List origins = new ArrayList<>(); JsonNode report = null; if (toolMetadataService.isMultiInput(step.operation())) { + // One call over all inputs. The outputs derive from a single input only when exactly + // one entered; otherwise (a genuine merge) there is no single source. ToolResult r = callEndpoint(step, inputFiles, supportingFiles); - files.addAll(r.files()); + Integer origin = inputOrigins.size() == 1 ? inputOrigins.get(0) : null; + for (Resource file : r.files()) { + files.add(file); + origins.add(origin); + } report = r.report(); } else if (inputFiles.isEmpty()) { ToolResult r = callEndpoint(step, List.of(), supportingFiles); - files.addAll(r.files()); + for (Resource file : r.files()) { + files.add(file); + origins.add(null); + } report = r.report(); } else { - for (Resource file : inputFiles) { - ToolResult r = callEndpoint(step, List.of(file), supportingFiles); - files.addAll(r.files()); + // One call per file: every output of this call inherits that input's origin, so a 1:1 + // op keeps its chain and a split (one input, many outputs) tags each output with the + // same source. + for (int k = 0; k < inputFiles.size(); k++) { + Integer origin = inputOrigins.get(k); + ToolResult r = callEndpoint(step, List.of(inputFiles.get(k)), supportingFiles); + for (Resource file : r.files()) { + files.add(file); + origins.add(origin); + } if (report == null) { report = r.report(); } } } - return new ToolResult(files, report); + return new StepOutput(files, origins, report); } /** diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java index b6d882bd6d..685ddb21db 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java @@ -1,5 +1,6 @@ package stirling.software.proprietary.policy.store; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -16,6 +17,8 @@ import stirling.software.proprietary.policy.model.Policy; public class InProcessPolicyStore implements PolicyStore { private final Map policies = new ConcurrentHashMap<>(); + // Run-order position per policy id, mirroring JpaPolicyStore's sort_order column. + private final Map sortOrders = new ConcurrentHashMap<>(); @Override public Policy save(Policy policy) { @@ -35,9 +38,25 @@ public class InProcessPolicyStore implements PolicyStore { policy.output(), policy.teamId()); policies.put(id, stored); + // Existing policy keeps its position; a new one appends to the end of its team's queue. + sortOrders.computeIfAbsent(id, key -> nextSortOrder(stored.teamId())); return stored; } + private int nextSortOrder(Long teamId) { + return policies.values().stream() + .filter(policy -> Objects.equals(policy.teamId(), teamId)) + .map(policy -> sortOrders.getOrDefault(policy.id(), 0)) + .max(Comparator.naturalOrder()) + .orElse(-1) + + 1; + } + + private Comparator byRunOrder() { + return Comparator.comparingInt(policy -> sortOrders.getOrDefault(policy.id(), 0)) + .thenComparing(Policy::id); + } + @Override public Optional get(String id) { return Optional.ofNullable(policies.get(id)); @@ -45,13 +64,14 @@ public class InProcessPolicyStore implements PolicyStore { @Override public List all() { - return List.copyOf(policies.values()); + return policies.values().stream().sorted(byRunOrder()).toList(); } @Override public List findByTeam(Long teamId) { return policies.values().stream() .filter(policy -> Objects.equals(policy.teamId(), teamId)) + .sorted(byRunOrder()) .toList(); } @@ -64,8 +84,21 @@ public class InProcessPolicyStore implements PolicyStore { .toList(); } + @Override + public void reorder(Long teamId, List orderedIds) { + int position = 0; + for (String id : orderedIds) { + Policy policy = policies.get(id); + if (policy == null || !Objects.equals(policy.teamId(), teamId)) { + continue; + } + sortOrders.put(id, position++); + } + } + @Override public boolean delete(String id) { + sortOrders.remove(id); return policies.remove(id) != null; } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java index 4de6ee2f3a..5089b8d419 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java @@ -1,11 +1,13 @@ package stirling.software.proprietary.policy.store; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import lombok.RequiredArgsConstructor; @@ -26,6 +28,7 @@ public class JpaPolicyStore implements PolicyStore { private final ObjectMapper objectMapper; @Override + @Transactional public Policy save(Policy policy) { String id = policy.id() == null || policy.id().isBlank() @@ -50,11 +53,49 @@ public class JpaPolicyStore implements PolicyStore { entity.setEnabled(stored.enabled()); entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type()); entity.setTeamId(stored.teamId()); + // Preserve an existing policy's run-order position; append a new one to the end of its + // team's queue (max + 1), so setting up a policy adds it last by default. + entity.setSortOrder( + repository + .findById(id) + .map(PolicyEntity::getSortOrder) + .orElseGet(() -> nextSortOrder(stored.teamId()))); entity.setPolicyJson(objectMapper.writeValueAsString(stored)); repository.save(entity); return stored; } + /** + * Append position for a new policy: max(existing) + 1, computed under a pessimistic lock on the + * team's rows (see {@link PolicyRepository#findByTeamForUpdate}) so two concurrent creates + * can't both read the same max and assign a duplicate order. (A brand-new team has no rows to + * lock; a rare simultaneous first-create there ties at 0 — harmless, since the ordering query + * breaks ties by id and any later reorder normalises it.) + */ + private int nextSortOrder(Long teamId) { + return repository.findByTeamForUpdate(teamId).stream() + .map(entity -> entity.getSortOrder() == null ? 0 : entity.getSortOrder()) + .max(Integer::compareTo) + .orElse(-1) + + 1; + } + + @Override + @Transactional + public void reorder(Long teamId, List orderedIds) { + int position = 0; + for (String id : orderedIds) { + PolicyEntity entity = repository.findById(id).orElse(null); + // Ignore unknown ids and any policy outside the caller's team — a reorder can't reach + // across teams. + if (entity == null || !Objects.equals(entity.getTeamId(), teamId)) { + continue; + } + entity.setSortOrder(position++); + repository.save(entity); + } + } + @Override public Optional get(String id) { return repository.findById(id).map(this::toPolicy); @@ -62,7 +103,7 @@ public class JpaPolicyStore implements PolicyStore { @Override public List all() { - return repository.findAll().stream().map(this::toPolicy).toList(); + return repository.findAllOrdered().stream().map(this::toPolicy).toList(); } @Override diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java index 4b99a90e6c..08e6bd4fbe 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java @@ -47,6 +47,14 @@ public class PolicyEntity implements Serializable { @Column(name = "team_id") private Long teamId; + /** + * Position in the team's run order (ascending). Team-wide and admin-editable; the per-trigger + * order the UI shows is this single sequence filtered by trigger. New policies are appended + * (max + 1). Nullable for pre-existing rows; treated as 0 when sorting. + */ + @Column(name = "sort_order") + private Integer sortOrder; + @Column(name = "policy_json", columnDefinition = "text") private String policyJson; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java index 8cb36b73d8..82ae4355dd 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java @@ -3,10 +3,13 @@ package stirling.software.proprietary.policy.store; import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import jakarta.persistence.LockModeType; + @Repository public interface PolicyRepository extends JpaRepository { @@ -14,12 +17,29 @@ public interface PolicyRepository extends JpaRepository { List findByTriggerTypeAndEnabledTrue(String triggerType); /** - * Policies belonging to a team, loaded without scanning every team's rows. A {@code null} - * teamId matches the rows with no team (login-disabled / pre-team data), mirroring the - * in-memory team filter rather than the empty result a plain {@code = null} would give. + * Policies belonging to a team, in run order (ascending {@code sortOrder}; a null order sorts + * first, id breaks ties for stability). A {@code null} teamId matches the rows with no team + * (login-disabled / pre-team data), mirroring the in-memory team filter rather than the empty + * result a plain {@code = null} would give. */ + @Query( + "select p from PolicyEntity p where ((:teamId is null and p.teamId is null) or" + + " p.teamId = :teamId) order by coalesce(p.sortOrder, 0) asc, p.id asc") + List findByTeam(@Param("teamId") Long teamId); + + /** All policies in run order — used when team scoping is off (login-disabled). */ + @Query("select p from PolicyEntity p order by coalesce(p.sortOrder, 0) asc, p.id asc") + List findAllOrdered(); + + /** + * The team's policy rows, locked for the transaction (SELECT … FOR UPDATE). Appending a new + * policy reads the max {@code sortOrder} from these under the lock, so two concurrent creates + * serialize instead of both reading a stale max and assigning the same position. Must be called + * inside a transaction. + */ + @Lock(LockModeType.PESSIMISTIC_WRITE) @Query( "select p from PolicyEntity p where (:teamId is null and p.teamId is null) or" + " p.teamId = :teamId") - List findByTeam(@Param("teamId") Long teamId); + List findByTeamForUpdate(@Param("teamId") Long teamId); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java index 67796d4a67..ac9e210439 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java @@ -21,6 +21,13 @@ public interface PolicyStore { /** Enabled policies with the given trigger type, for background triggers. */ List findByTriggerType(String triggerType); + /** + * Set the team's run order from {@code orderedIds} (position → sortOrder). Only policies that + * belong to {@code teamId} are touched; unknown/other-team ids are ignored, so a caller can't + * reorder across teams. Ids omitted from the list keep their existing order. + */ + void reorder(Long teamId, List orderedIds); + /** Returns whether the policy existed. */ boolean delete(String id); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java index c53d835ca5..74f904c455 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java @@ -36,7 +36,8 @@ import stirling.software.common.model.exception.UnsupportedProviderException; "stirling.software.proprietary.policy.source", "stirling.software.proprietary.accountlink", "stirling.software.proprietary.access.repository", - "stirling.software.proprietary.integration.repository" + "stirling.software.proprietary.integration.repository", + "stirling.software.proprietary.classification.store" }) @EntityScan({ "stirling.software.proprietary.security.model", @@ -47,7 +48,8 @@ import stirling.software.common.model.exception.UnsupportedProviderException; "stirling.software.proprietary.policy.source", "stirling.software.proprietary.accountlink", "stirling.software.proprietary.access.model", - "stirling.software.proprietary.integration.model" + "stirling.software.proprietary.integration.model", + "stirling.software.proprietary.classification.store" }) public class DatabaseConfig { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index d3b2d21a00..0e9460eca7 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -404,6 +404,7 @@ public class AiWorkflowService { buildCompletedResponse( response.getRationale(), result.files(), + result.origins(), inputFileNames(filesById), result.report())); } catch (InternalApiTimeoutException e) { @@ -465,7 +466,8 @@ public class AiWorkflowService { } }; return new WorkflowState.Terminal( - buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null)); + buildCompletedResponse( + response.getSummary(), List.of(resource), null, List.of(), null)); } @SuppressWarnings("unchecked") @@ -523,7 +525,11 @@ public class AiWorkflowService { return new WorkflowState.Terminal( buildCompletedResponse( - summary, result.files(), inputFileNames(filesById), result.report())); + summary, + result.files(), + result.origins(), + inputFileNames(filesById), + result.report())); } catch (InternalApiTimeoutException e) { log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage()); return new WorkflowState.Terminal( @@ -612,19 +618,35 @@ public class AiWorkflowService { private AiWorkflowResponse buildCompletedResponse( String summary, List resultFiles, + List origins, List inputFileNames, JsonNode report) throws IOException { // Store every output file individually so each gets its own Stirling file ID and the // frontend can add them as independent variants without going through a zip. - boolean preserveInputNames = inputFileNames.size() == resultFiles.size(); + // Count outputs per source so only a clean 1:1 transform (one output for a source) reuses + // the input's name; a split (one input → many outputs) keeps each entry's own name. + Map outputsPerOrigin = + origins == null + ? Map.of() + : origins.stream() + .filter(o -> o != null) + .collect(Collectors.groupingBy(o -> o, Collectors.counting())); List descriptors = new ArrayList<>(); for (int i = 0; i < resultFiles.size(); i++) { Resource resource = resultFiles.get(i); String responseName = resource.getFilename(); - String inputName = preserveInputNames ? inputFileNames.get(i) : null; - // Prefer the input name only for 1:1 operations where the output keeps the same - // extension (rotate, compress, etc.). For converters and other extension-changing + // The output's source input (from the executor), used both to name it and to tell the + // client which file to version in place. + Integer origin = origins != null && i < origins.size() ? origins.get(i) : null; + boolean uniqueOrigin = + origin != null && outputsPerOrigin.getOrDefault(origin, 0L) == 1L; + String inputName = + uniqueOrigin && origin >= 0 && origin < inputFileNames.size() + ? inputFileNames.get(origin) + : null; + // Prefer the source input's name only for 1:1 operations where the output keeps the + // same extension (rotate, compress, etc.). For converters and other extension-changing // tools, the response filename from Content-Disposition is authoritative. String name; if (inputName != null @@ -644,7 +666,11 @@ public class AiWorkflowService { try (java.io.InputStream is = resource.getInputStream()) { fileId = fileStorage.storeInputStream(is, name).fileId(); } - descriptors.add(new AiWorkflowResultFile(fileId, name, contentType)); + // Only expose the source when this is a clean 1:1 transform, so the client can treat a + // present sourceIndex as "replace that input in place" without further disambiguation. + descriptors.add( + new AiWorkflowResultFile( + fileId, name, contentType, uniqueOrigin ? origin : null)); } AiWorkflowResponse completed = new AiWorkflowResponse(); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/classification/ClassificationLabelsControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/classification/ClassificationLabelsControllerTest.java new file mode 100644 index 0000000000..221d567c31 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/classification/ClassificationLabelsControllerTest.java @@ -0,0 +1,133 @@ +package stirling.software.proprietary.classification; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.server.ResponseStatusException; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.UserServiceInterface; +import stirling.software.proprietary.classification.model.ClassificationLabel; +import stirling.software.proprietary.classification.model.ClassificationLabels; +import stirling.software.proprietary.classification.store.ClassificationLabelStore; +import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; + +@ExtendWith(MockitoExtension.class) +@DisplayName("ClassificationLabelsController") +class ClassificationLabelsControllerTest { + + private static final Long TEAM = 7L; + + @Mock private PolicyManagementAuthority policyManagementAuthority; + @Mock private UserServiceInterface userService; + + private ClassificationLabelStore store; + private ApplicationProperties applicationProperties; + private ClassificationLabelsController controller; + + @BeforeEach + void setUp() { + store = new InProcessClassificationLabelStore(); + applicationProperties = new ApplicationProperties(); + controller = + new ClassificationLabelsController( + store, policyManagementAuthority, applicationProperties, userService); + } + + private static ClassificationLabels sample() { + return new ClassificationLabels( + List.of( + new ClassificationLabel("invoice", "Invoice", "receipt-long"), + new ClassificationLabel("contract", "Contract", null))); + } + + private void loginEnabled(boolean enabled) { + applicationProperties.getSecurity().setEnableLogin(enabled); + } + + @Test + @DisplayName("GET returns 204 when the team has no labels") + void getEmpty() { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + ResponseEntity response = controller.getTeamLabels(); + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + } + + @Test + @DisplayName("PUT then GET round-trips the team's labels (login disabled)") + void saveThenGet() { + loginEnabled(false); + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + + controller.saveTeamLabels(sample()); + ResponseEntity got = controller.getTeamLabels(); + + assertThat(got.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(got.getBody()).isNotNull(); + assertThat(got.getBody().labels()).hasSize(2); + assertThat(got.getBody().labels().getFirst().name()).isEqualTo("Invoice"); + assertThat(got.getBody().labels().getFirst().icon()).isEqualTo("receipt-long"); + } + + @Test + @DisplayName("PUT is scoped per team") + void perTeam() { + loginEnabled(false); + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + controller.saveTeamLabels(sample()); + + when(policyManagementAuthority.currentUserTeamId()).thenReturn(99L); + assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + } + + @Test + @DisplayName("PUT is rejected for a non-editor when login is enabled") + void putForbiddenForNonEditor() { + loginEnabled(true); + when(policyManagementAuthority.canEditPolicies()).thenReturn(false); + + assertThatThrownBy(() -> controller.saveTeamLabels(sample())) + .isInstanceOf(ResponseStatusException.class) + .hasFieldOrPropertyWithValue("statusCode", HttpStatus.FORBIDDEN); + } + + @Test + @DisplayName("PUT rejects an invalid label set with 400") + void putInvalid() { + loginEnabled(false); + ClassificationLabels duplicate = + new ClassificationLabels( + List.of( + new ClassificationLabel("invoice", "Invoice", null), + new ClassificationLabel("invoice", "Invoice", null))); + + assertThatThrownBy(() -> controller.saveTeamLabels(duplicate)) + .isInstanceOf(ResponseStatusException.class) + .hasFieldOrPropertyWithValue("statusCode", HttpStatus.BAD_REQUEST); + } + + @Test + @DisplayName("DELETE resets the team back to no stored labels") + void deleteResets() { + loginEnabled(false); + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + controller.saveTeamLabels(sample()); + + ResponseEntity response = controller.resetTeamLabels(); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/classification/model/LabelsValidatorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/classification/model/LabelsValidatorTest.java new file mode 100644 index 0000000000..b9730370c7 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/classification/model/LabelsValidatorTest.java @@ -0,0 +1,146 @@ +package stirling.software.proprietary.classification.model; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import java.util.Locale; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("LabelsValidator") +class LabelsValidatorTest { + + private static ClassificationLabels labels(ClassificationLabel... labels) { + return new ClassificationLabels(List.of(labels)); + } + + private static ClassificationLabel label(String name) { + return new ClassificationLabel(slug(name), name, null); + } + + private static String slug(String name) { + return name.trim() + .toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9]+", "-") + .replaceAll("(^-|-$)", ""); + } + + @Test + @DisplayName("accepts a well-formed label set") + void acceptsValid() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice", "Invoice", "receipt-long"), + label("Contract")); + assertThatCode(() -> LabelsValidator.validate(set)).doesNotThrowAnyException(); + } + + @Test + @DisplayName("accepts an empty label set (reads as: use the default)") + void acceptsEmpty() { + assertThatCode(() -> LabelsValidator.validate(labels())).doesNotThrowAnyException(); + } + + @Test + @DisplayName("rejects a null label set") + void rejectsNull() { + assertThatThrownBy(() -> LabelsValidator.validate(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Labels are required"); + } + + @Test + @DisplayName("rejects duplicate names (distinct ids)") + void rejectsDuplicateNames() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice-a", "Invoice", null), + new ClassificationLabel("invoice-b", "Invoice", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Duplicate label name"); + } + + @Test + @DisplayName("rejects duplicate names differing only by case") + void rejectsDuplicateNamesCaseInsensitive() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice-a", "Invoice", null), + new ClassificationLabel("invoice-b", "INVOICE", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Duplicate label name"); + } + + @Test + @DisplayName("rejects duplicate ids") + void rejectsDuplicateIds() { + ClassificationLabels set = + labels( + new ClassificationLabel("invoice", "Invoice", null), + new ClassificationLabel("invoice", "Sales invoice", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Duplicate label id"); + } + + @Test + @DisplayName("rejects a blank name") + void rejectsBlankName() { + ClassificationLabels set = labels(new ClassificationLabel("blank", " ", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Label name must not be blank"); + } + + @Test + @DisplayName("rejects a blank id") + void rejectsBlankId() { + ClassificationLabels set = labels(new ClassificationLabel(" ", "Invoice", null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Label id must not be blank"); + } + + @Test + @DisplayName("rejects an over-long name") + void rejectsOverLongName() { + ClassificationLabels set = + labels( + new ClassificationLabel( + "x", "x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1), null)); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("too long"); + } + + @Test + @DisplayName("rejects an over-long icon (a null icon is fine)") + void rejectsOverLongIcon() { + ClassificationLabels set = + labels( + new ClassificationLabel( + "invoice", + "Invoice", + "x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1))); + assertThatThrownBy(() -> LabelsValidator.validate(set)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("icon is too long"); + } + + @Test + @DisplayName("rejects more labels than the cap") + void rejectsTooManyLabels() { + List tooMany = + IntStream.rangeClosed(0, LabelsValidator.MAX_LABELS) + .mapToObj(i -> label("label" + i)) + .toList(); + assertThatThrownBy(() -> LabelsValidator.validate(new ClassificationLabels(tooMany))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Too many labels"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java new file mode 100644 index 0000000000..c5ee39be14 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java @@ -0,0 +1,176 @@ +package stirling.software.proprietary.controller.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.web.multipart.MultipartFile; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.classification.model.ClassificationLabel; +import stirling.software.proprietary.classification.model.ClassificationLabels; +import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore; +import stirling.software.proprietary.policy.config.PolicyManagementAuthority; +import stirling.software.proprietary.service.AiEngineClient; +import stirling.software.proprietary.service.PdfContentExtractor; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ClassifyLabelControllerTest { + + private static final Long TEAM = 7L; + + @Mock private CustomPDFDocumentFactory pdfDocumentFactory; + @Mock private TempFileManager tempFileManager; + @Mock private PdfContentExtractor pdfContentExtractor; + @Mock private PdfMetadataService pdfMetadataService; + @Mock private AiEngineClient aiEngineClient; + @Mock private PolicyManagementAuthority policyManagementAuthority; + + private final ObjectMapper objectMapper = JsonMapper.builder().build(); + private InProcessClassificationLabelStore labelStore; + private ClassifyLabelController controller; + + @BeforeEach + void setUp() { + labelStore = new InProcessClassificationLabelStore(); + controller = + new ClassifyLabelController( + pdfDocumentFactory, + tempFileManager, + pdfContentExtractor, + pdfMetadataService, + aiEngineClient, + objectMapper, + null, + labelStore, + policyManagementAuthority); + } + + private void stubSinglePageDocument() throws Exception { + PDDocument document = mock(PDDocument.class); + MultipartFile file = mock(MultipartFile.class); + when(file.getOriginalFilename()).thenReturn("invoice.pdf"); + when(pdfDocumentFactory.load(any(MultipartFile.class), eq(true))).thenReturn(document); + when(document.getNumberOfPages()).thenReturn(1); + when(pdfContentExtractor.extractPageTextRaw(document, 1)) + .thenReturn("Invoice total due 100.00"); + when(aiEngineClient.post(eq("/api/v1/documents/classify"), anyString(), isNull())) + .thenReturn("{\"outcome\":\"classification\",\"labels\":[\"invoice\"]}"); + + try { + controller.classifyAndLabel(file); + } catch (Exception ignored) { + // WebResponseUtils.pdfDocToWebResponse needs a real temp file; the engine call and + // metadata write we assert on have already happened by the time it runs. + } + } + + private JsonNode sentEngineRequest() throws Exception { + ArgumentCaptor body = ArgumentCaptor.forClass(String.class); + verify(aiEngineClient).post(eq("/api/v1/documents/classify"), body.capture(), isNull()); + return objectMapper.readTree(body.getValue()); + } + + @Test + void classifyAndLabel_writesClassificationWithoutOutcome() throws Exception { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + labelStore.save( + TEAM, + new ClassificationLabels( + List.of(new ClassificationLabel("invoice", "Invoice", null))), + "admin"); + + stubSinglePageDocument(); + + ArgumentCaptor value = ArgumentCaptor.forClass(String.class); + verify(pdfMetadataService) + .setClassificationMetadata(any(PDDocument.class), value.capture()); + + JsonNode written = objectMapper.readTree(value.getValue()); + assertThat(written.has("outcome")).isFalse(); + // The engine returns label ids; they're stored on the document verbatim. + assertThat(written.get("labels").get(0).asText()).isEqualTo("invoice"); + } + + @Test + void classifyAndLabel_sendsTeamLabelIdsAndNames() throws Exception { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + labelStore.save( + TEAM, + new ClassificationLabels( + List.of( + new ClassificationLabel("invoice", "Invoice", "receipt-long"), + new ClassificationLabel("contract", "Contract", null), + new ClassificationLabel("timesheet", "Timesheet", null))), + "admin"); + + stubSinglePageDocument(); + + JsonNode request = sentEngineRequest(); + assertThat(request.get("fileName").asText()).isEqualTo("invoice.pdf"); + JsonNode labels = request.get("labels"); + assertThat(labels.isArray()).isTrue(); + assertThat(labels.size()).isEqualTo(3); + // Each entry is an {id, name} pair — the model reasons over names, we store ids. + assertThat( + List.of( + labels.get(0).get("id").asText(), + labels.get(1).get("id").asText(), + labels.get(2).get("id").asText())) + .containsExactly("invoice", "contract", "timesheet"); + assertThat( + List.of( + labels.get(0).get("name").asText(), + labels.get(1).get("name").asText(), + labels.get(2).get("name").asText())) + .containsExactly("Invoice", "Contract", "Timesheet"); + } + + @Test + void classifyAndLabel_skipsClassificationWhenNothingStored() throws Exception { + when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM); + + stubSinglePageDocument(); + + // No team labels stored, and the engine holds no default of its own, so the file is passed + // through unlabelled: neither the engine nor the metadata write is invoked. + verify(aiEngineClient, never()).post(anyString(), anyString(), any()); + verify(pdfMetadataService, never()) + .setClassificationMetadata(any(PDDocument.class), anyString()); + } + + @Test + void windowPageNumbers_takesFirstAndLastWithoutOverlap() { + assertEquals(List.of(1, 2, 4, 5), ClassifyLabelController.windowPageNumbers(5, 2)); + assertEquals(List.of(1, 2, 3), ClassifyLabelController.windowPageNumbers(3, 2)); + // Short docs clamp + dedupe rather than throwing or going out of range. + assertEquals(List.of(1, 2), ClassifyLabelController.windowPageNumbers(2, 2)); + assertEquals(List.of(1), ClassifyLabelController.windowPageNumbers(1, 2)); + assertEquals(List.of(), ClassifyLabelController.windowPageNumbers(0, 2)); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index 7dc174c041..15c07bab71 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -2,6 +2,7 @@ package stirling.software.proprietary.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -230,6 +231,57 @@ class AiWorkflowServiceTest { // 1:1 mapping preserves each input's filename. assertEquals("a.pdf", result.getResultFiles().get(0).getFileName()); assertEquals("b.pdf", result.getResultFiles().get(1).getFileName()); + // Each output points back at the input it came from so the client versions it in place. + assertEquals(0, result.getResultFiles().get(0).getSourceIndex()); + assertEquals(1, result.getResultFiles().get(1).getSourceIndex()); + } + + @Test + void mergeOutputHasNoSourceIndex() throws IOException { + MockMultipartFile a = pdf("a.pdf", "a-bytes"); + MockMultipartFile b = pdf("b.pdf", "b-bytes"); + stubOrchestrator( + """ + {"outcome":"tool_call","tool":"%s","parameters":{},"rationale":"Merging"} + """ + .formatted(MERGE_ENDPOINT)); + when(toolMetadataService.isMultiInput(MERGE_ENDPOINT)).thenReturn(true); + when(toolMetadataService.shouldUnpackZipResponse(MERGE_ENDPOINT)).thenReturn(false); + stubEndpoint(MERGE_ENDPOINT, pdfResource("merged-bytes", "merged.pdf")); + stubFileStorage(); + + AiWorkflowResponse result = + service.orchestrate(requestFor(new MockMultipartFile[] {a, b}, "merge these")); + + // A merge draws on several inputs, so there is no single source to version in place. + assertNull(result.getResultFiles().get(0).getSourceIndex()); + } + + @Test + void splitOutputsHaveNoSourceIndex() throws IOException { + MockMultipartFile input = pdf("doc.pdf", "original"); + stubOrchestrator( + """ + {"outcome":"tool_call","tool":"%s","parameters":{},"rationale":"Splitting"} + """ + .formatted(SPLIT_ENDPOINT)); + when(toolMetadataService.isMultiInput(SPLIT_ENDPOINT)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(SPLIT_ENDPOINT)).thenReturn(true); + stubEndpoint( + SPLIT_ENDPOINT, + zipResource( + "doc.zip", + List.of( + new ZipEntryBytes("page-1.pdf", "page-one"), + new ZipEntryBytes("page-2.pdf", "page-two")))); + stubFileStorage(); + + AiWorkflowResponse result = service.orchestrate(requestFor(input, "split")); + + // One input fanned out to many outputs, so none is a clean 1:1 version — the client adds + // them as fresh files and leaves the original in place. + assertNull(result.getResultFiles().get(0).getSourceIndex()); + assertNull(result.getResultFiles().get(1).getSourceIndex()); } @Test diff --git a/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java b/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java index d93eacbafa..5263705986 100644 --- a/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java +++ b/app/saas/src/main/java/stirling/software/saas/security/TeamSecurityExpressions.java @@ -65,6 +65,12 @@ public class TeamSecurityExpressions { return currentUser.getTeam().getId(); } + /** The current authenticated user's id, or {@code null} if unauthenticated. */ + public Long currentUserId() { + User currentUser = getCurrentUser(); + return currentUser == null ? null : currentUser.getId(); + } + /** Whether the current authenticated user is any kind of member of the given team. */ public boolean isTeamMember(Long teamId) { User currentUser = getCurrentUser(); diff --git a/app/saas/src/main/resources/db/migration/saas/V30__classification_labels.sql b/app/saas/src/main/resources/db/migration/saas/V30__classification_labels.sql new file mode 100644 index 0000000000..0719d73fb1 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V30__classification_labels.sql @@ -0,0 +1,13 @@ +-- Classification labels (gated by policies.enabled): the flat multi-label vocabulary the document +-- classifier runs against. One admin-editable row per team. The whole label set lives as JSON in +-- labels_json (authoritative on read). team_id is a natural key and a plain value (not a foreign +-- key) to stay decoupled from the security entities, so classification can be enabled or disabled +-- without touching them; the sentinel 0 holds the unteamed (login-disabled) team set. Hibernate +-- ddl-auto would also create this, but this keeps the schema explicit for Flyway-managed deploys. + +CREATE TABLE IF NOT EXISTS classification_labels ( + team_id BIGINT PRIMARY KEY, + labels_json TEXT, + updated_at TIMESTAMP, + updated_by VARCHAR(255) +); diff --git a/app/saas/src/main/resources/db/migration/saas/V31__policy_sort_order.sql b/app/saas/src/main/resources/db/migration/saas/V31__policy_sort_order.sql new file mode 100644 index 0000000000..2b2bdbd25b --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V31__policy_sort_order.sql @@ -0,0 +1,5 @@ +-- Team-wide policy run order (see PolicyEntity.sortOrder). The order policies run in is stored +-- server-side and shared by the whole team (not per-user in the browser), and is admin-editable. +-- Nullable so existing rows keep working; reads treat a null as 0 (coalesce), and the store +-- appends a new policy at max(order)+1 so setting one up adds it to the end of the queue. +ALTER TABLE policies ADD COLUMN sort_order INTEGER; diff --git a/engine/src/stirling/agents/__init__.py b/engine/src/stirling/agents/__init__.py index c22bd6c977..7bc2cb63a2 100644 --- a/engine/src/stirling/agents/__init__.py +++ b/engine/src/stirling/agents/__init__.py @@ -1,5 +1,6 @@ """Agent modules for Stirling AI reasoning flows.""" +from .document_classifier import DocumentClassifierAgent from .execution import ExecutionPlanningAgent from .orchestrator import OrchestratorAgent from .pdf_create import PdfCreateAgent @@ -9,6 +10,7 @@ from .pdf_review import PdfReviewAgent from .user_spec import UserSpecAgent __all__ = [ + "DocumentClassifierAgent", "ExecutionPlanningAgent", "OrchestratorAgent", "PdfCreateAgent", diff --git a/engine/src/stirling/agents/document_classifier.py b/engine/src/stirling/agents/document_classifier.py new file mode 100644 index 0000000000..bb4ab5ef00 --- /dev/null +++ b/engine/src/stirling/agents/document_classifier.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import logging + +from pydantic import Field +from pydantic_ai import Agent +from pydantic_ai.output import NativeOutput + +from stirling.contracts import ( + ClassifyDocumentRequest, + ClassifyDocumentResponse, + DocumentClassificationResponse, + LabelOption, + PageText, +) +from stirling.models import ApiModel +from stirling.services import AppRuntime + +logger = logging.getLogger(__name__) + +# A document carries at most this many labels; anything past the cap is dropped +# in the model's order. +MAX_ASSIGNED_LABELS = 5 +# Pages read from each end of the document. A document's type is evident from +# its opening (and closing) pages, so a fixed window keeps cost and latency flat +# regardless of length. Promote to AppSettings if it ever needs tuning. +WINDOW_PAGES = 2 + + +_SYSTEM_PROMPT = ( + "You identify what a document is by assigning labels, choosing only from a " + "fixed list of allowed labels you are given.\n" + "\n" + "Rules:\n" + f"- Pick up to {MAX_ASSIGNED_LABELS} labels that describe this document's type.\n" + "- Only use labels from the allowed list, spelled exactly as listed.\n" + "- Return an empty list if none fit.\n" + "- Judge from the document's content and structure, not from keywords alone. " + "The document may be in any language.\n" + "- You are shown only the first and last pages; that is enough to identify the type." +) + + +class _ClassifierOutput(ApiModel): + """Raw model answer, before it is validated against the allowed vocabulary.""" + + labels: list[str] = Field( + default_factory=list, + description="Labels from the allowed list that describe this document's type.", + ) + + +def render_labels(labels: list[LabelOption]) -> str: + """Render the allowed vocabulary for the prompt — the model reasons over the + human names (it never sees ids).""" + names = [label.name for label in labels] + return f"Allowed labels: {', '.join(names) or '(none)'}" + + +def select_window(pages: list[PageText], window: int = WINDOW_PAGES) -> list[PageText]: + """Return the first and last ``window`` pages, never overlapping. + + Documents short enough that the two ends would meet are returned whole. The + caller usually sends just the window already; this is a defensive trim in + case it sends more. + """ + if window <= 0 or len(pages) <= window * 2: + return list(pages) + return [*pages[:window], *pages[-window:]] + + +def format_window(pages: list[PageText]) -> str: + if not pages: + return "(no extractable text)" + return "\n\n".join(f"[Page {page.page_number}]\n{page.text}" for page in pages) + + +def validate_labels(output: _ClassifierOutput, allowed: list[LabelOption]) -> DocumentClassificationResponse: + """Coerce a raw model answer (names) onto the allowed vocabulary, returning + label **ids**. + + The model answers with names; they are matched case-insensitively to the + allowed vocabulary and returned as that label's id. Anything off-list is + dropped, duplicates collapse to the first occurrence, and the result is + capped at ``MAX_ASSIGNED_LABELS`` in the model's order. The model identifies; + these rules decide what is allowed to stand. + """ + id_by_lower_name = {label.name.lower(): label.id for label in allowed} + + kept: list[str] = [] + for name in output.labels: + label_id = id_by_lower_name.get(name.strip().lower()) + if label_id is not None and label_id not in kept: + kept.append(label_id) + if len(kept) == MAX_ASSIGNED_LABELS: + break + + return DocumentClassificationResponse(labels=kept) + + +class DocumentClassifierAgent: + """Assigns labels to a document from an allowed vocabulary. + + Reads the bounded page window supplied on the request (first/last + ``WINDOW_PAGES``) and runs a single fast-model pass, then validates the + answer against the vocabulary so nothing off-list survives. + """ + + def __init__(self, runtime: AppRuntime) -> None: + self.runtime = runtime + self._agent = Agent( + model=runtime.fast_model, + output_type=NativeOutput(_ClassifierOutput), + system_prompt=_SYSTEM_PROMPT, + model_settings=runtime.fast_model_settings, + ) + + async def classify(self, request: ClassifyDocumentRequest) -> ClassifyDocumentResponse: + # The caller (the backend) always supplies the allowed vocabulary — the + # team's stored labels — so the engine holds no vocabulary of its own. + allowed = request.labels + window = select_window(request.pages) + prompt = self._build_prompt(request.file_name, allowed, window) + logger.debug("[classify] prompt:\n%s", prompt) + result = await self._agent.run(prompt) + return validate_labels(result.output, allowed) + + @staticmethod + def _build_prompt(file_name: str, allowed: list[LabelOption], window: list[PageText]) -> str: + return ( + f"{render_labels(allowed)}\n\n" + f"Document file name: {file_name}\n" + f"Document content (first and last pages):\n{format_window(window)}" + ) diff --git a/engine/src/stirling/api/app.py b/engine/src/stirling/api/app.py index 7e6272c28b..34b0da36fd 100644 --- a/engine/src/stirling/api/app.py +++ b/engine/src/stirling/api/app.py @@ -10,6 +10,7 @@ from pydantic_ai import Agent from pydantic_ai.models.instrumented import InstrumentationSettings from stirling.agents import ( + DocumentClassifierAgent, ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, @@ -24,6 +25,7 @@ from stirling.api.middleware import UserIdMiddleware from stirling.api.routes import ( agent_capabilities_router, agent_draft_router, + document_classifier_router, document_router, execution_router, ledger_router, @@ -95,6 +97,7 @@ async def lifespan(fast_api: FastAPI): fast_api.state.execution_planning_agent = ExecutionPlanningAgent(runtime) fast_api.state.math_auditor_agent = MathAuditorAgent(runtime) fast_api.state.pdf_comment_agent = PdfCommentAgent(runtime) + fast_api.state.document_classifier_agent = DocumentClassifierAgent(runtime) tracer_provider = setup_posthog_tracking(settings) if tracer_provider: Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider)) @@ -131,6 +134,7 @@ app.include_router(document_router, dependencies=_user_gate) app.include_router(ledger_router, dependencies=_user_gate) app.include_router(pdf_comments_router, dependencies=_user_gate) app.include_router(agent_capabilities_router, dependencies=_user_gate) +app.include_router(document_classifier_router, dependencies=_user_gate) @app.get("/health", response_model=HealthResponse) diff --git a/engine/src/stirling/api/dependencies.py b/engine/src/stirling/api/dependencies.py index 780e97c9d7..ed9aa3f041 100644 --- a/engine/src/stirling/api/dependencies.py +++ b/engine/src/stirling/api/dependencies.py @@ -5,6 +5,7 @@ from typing import Annotated from fastapi import Depends, HTTPException, Request, status from stirling.agents import ( + DocumentClassifierAgent, ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, @@ -55,6 +56,10 @@ def get_pdf_comment_agent(request: Request) -> PdfCommentAgent: return request.app.state.pdf_comment_agent +def get_document_classifier_agent(request: Request) -> DocumentClassifierAgent: + return request.app.state.document_classifier_agent + + def require_user_id() -> UserId: """FastAPI dependency for routes that touch per-user storage. diff --git a/engine/src/stirling/api/routes/__init__.py b/engine/src/stirling/api/routes/__init__.py index 1c1e4e1d23..f40e9918f8 100644 --- a/engine/src/stirling/api/routes/__init__.py +++ b/engine/src/stirling/api/routes/__init__.py @@ -1,5 +1,6 @@ from .agent_capabilities import router as agent_capabilities_router from .agent_drafts import router as agent_draft_router +from .document_classifier import router as document_classifier_router from .documents import router as document_router from .execution import router as execution_router from .ledger import router as ledger_router @@ -11,6 +12,7 @@ from .pdf_questions import router as pdf_question_router __all__ = [ "agent_capabilities_router", "agent_draft_router", + "document_classifier_router", "document_router", "execution_router", "ledger_router", diff --git a/engine/src/stirling/api/routes/document_classifier.py b/engine/src/stirling/api/routes/document_classifier.py new file mode 100644 index 0000000000..649c0f4e9e --- /dev/null +++ b/engine/src/stirling/api/routes/document_classifier.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Annotated + +from fastapi import APIRouter, Depends + +from stirling.agents import DocumentClassifierAgent +from stirling.api.dependencies import get_document_classifier_agent +from stirling.contracts import ClassifyDocumentRequest, ClassifyDocumentResponse + +router = APIRouter(prefix="/api/v1/documents/classify", tags=["document-classifier"]) + + +@router.post("", response_model=ClassifyDocumentResponse) +async def classify_document( + request: ClassifyDocumentRequest, + agent: Annotated[DocumentClassifierAgent, Depends(get_document_classifier_agent)], +) -> ClassifyDocumentResponse: + """Classify a document from its supplied page text against the allowed labels. + + The caller sends the bounded page window inline, so no per-user document + storage is touched here — the request is self-contained. + """ + return await agent.classify(request) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 77ce40301e..513906d610 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -35,6 +35,12 @@ from .contradiction import ( ContradictionReport, ContradictionSeverity, ) +from .document_classifier import ( + ClassifyDocumentRequest, + ClassifyDocumentResponse, + DocumentClassificationResponse, + LabelOption, +) from .documents import ( DeleteDocumentResponse, IngestDocumentRequest, @@ -128,6 +134,8 @@ __all__ = [ "AiToolAgentStep", "ArtifactKind", "CannotContinueExecutionAction", + "ClassifyDocumentRequest", + "ClassifyDocumentResponse", "Claim", "CommentSpec", "CompletedExecutionAction", @@ -138,6 +146,8 @@ __all__ = [ "DeleteDocumentResponse", "PurgeOwnerResponse", "Discrepancy", + "DocumentClassificationResponse", + "LabelOption", "DocumentMeta", "DocumentSections", "DiscrepancyKind", diff --git a/engine/src/stirling/contracts/document_classifier.py b/engine/src/stirling/contracts/document_classifier.py new file mode 100644 index 0000000000..d64e79f492 --- /dev/null +++ b/engine/src/stirling/contracts/document_classifier.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from pydantic import Field + +from stirling.models import ApiModel + +from .documents import PageText + + +class LabelOption(ApiModel): + """One allowed label: a stable ``id`` (the identity) plus the human ``name`` + the model actually reasons over. The model picks names; the engine maps its + answer back to ids, so the result is language- and rename-stable.""" + + id: str = Field(min_length=1) + name: str = Field(min_length=1) + + +class ClassifyDocumentRequest(ApiModel): + """Classify one document from its page text. + + The caller sends the page text directly — typically just the bounded window + (first/last pages), since the classifier reads no more than that. There is no + ingestion or RAG step. + + ``labels`` is the allowed vocabulary for this request — the caller (the + backend) always supplies it from the team's stored label set, each an + ``{id, name}`` pair. The engine holds no vocabulary of its own, so at least + one label is required. + """ + + file_name: str = Field(min_length=1) + pages: list[PageText] = Field(default_factory=list) + labels: list[LabelOption] = Field(min_length=1) + + +class DocumentClassificationResponse(ApiModel): + """Terminal classification result. + + ``labels`` is the subset of the allowed vocabulary the model assigned to the + document, as label **ids**: at most five entries, deduplicated, in the + model's order. An empty list is a valid answer — nothing in the vocabulary + fit. This is a plain answer from a dedicated endpoint — it carries no + ``outcome`` discriminator (it isn't one of the orchestrator's + WorkflowOutcome-routed union responses). + """ + + labels: list[str] = Field(default_factory=list) + + +# Only one response shape today; kept as a named alias so routes and agents have +# a stable response type to import. +ClassifyDocumentResponse = DocumentClassificationResponse diff --git a/engine/tests/test_document_classifier.py b/engine/tests/test_document_classifier.py new file mode 100644 index 0000000000..869d0dc728 --- /dev/null +++ b/engine/tests/test_document_classifier.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from stirling.agents.document_classifier import ( + MAX_ASSIGNED_LABELS, + DocumentClassifierAgent, + _ClassifierOutput, + render_labels, + select_window, + validate_labels, +) +from stirling.contracts import ( + ClassifyDocumentRequest, + DocumentClassificationResponse, + LabelOption, + PageText, +) +from stirling.services.runtime import AppRuntime + + +def _page(number: int, text: str = "x") -> PageText: + return PageText(page_number=number, text=text) + + +def _slug(name: str) -> str: + return re.sub(r"^-+|-+$", "", re.sub(r"[^a-z0-9]+", "-", name.lower())) + + +def _opts(*names: str) -> list[LabelOption]: + """Allowed vocabulary from names, with slug ids (mirrors the frontend).""" + return [LabelOption(id=_slug(name), name=name) for name in names] + + +# ── select_window ─────────────────────────────────────────────────────────── + + +def test_select_window_returns_short_documents_whole() -> None: + pages = [_page(1), _page(2), _page(3), _page(4)] + assert select_window(pages, window=2) == pages + + +def test_select_window_takes_both_ends_without_overlap() -> None: + pages = [_page(n) for n in range(1, 6)] # 5 pages + selected = select_window(pages, window=2) + assert [p.page_number for p in selected] == [1, 2, 4, 5] + + +def test_select_window_handles_empty() -> None: + assert select_window([], window=2) == [] + + +def test_select_window_zero_returns_all() -> None: + pages = [_page(1), _page(2), _page(3)] + assert select_window(pages, window=0) == pages + + +# ── validate_labels ────────────────────────────────────────────────────────── + + +def test_valid_labels_are_preserved_in_order() -> None: + # Model answers with names; the result is the matching label ids. + output = _ClassifierOutput(labels=["Invoice", "Receipt"]) + result = validate_labels(output, _opts("Invoice", "Receipt", "Purchase order")) + assert isinstance(result, DocumentClassificationResponse) + assert result.labels == ["invoice", "receipt"] + + +def test_off_list_labels_are_dropped() -> None: + output = _ClassifierOutput(labels=["Invoice", "Warp core", "Receipt"]) + result = validate_labels(output, _opts("Invoice", "Receipt")) + assert result.labels == ["invoice", "receipt"] + + +def test_matching_is_case_insensitive_and_returns_ids() -> None: + output = _ClassifierOutput(labels=["invoice", " CREDIT NOTE "]) + result = validate_labels(output, _opts("Invoice", "Credit note")) + assert result.labels == ["invoice", "credit-note"] + + +def test_duplicates_collapse_to_first_occurrence() -> None: + output = _ClassifierOutput(labels=["Invoice", "invoice", "Receipt", "INVOICE"]) + result = validate_labels(output, _opts("Invoice", "Receipt")) + assert result.labels == ["invoice", "receipt"] + + +def test_result_is_capped_at_max_assigned_labels() -> None: + allowed = _opts(*[f"Label {n}" for n in range(10)]) + output = _ClassifierOutput(labels=[label.name for label in allowed]) + result = validate_labels(output, allowed) + assert result.labels == [label.id for label in allowed[:MAX_ASSIGNED_LABELS]] + + +def test_empty_answer_is_valid() -> None: + result = validate_labels(_ClassifierOutput(labels=[]), _opts("Invoice")) + assert result.labels == [] + + +def test_entirely_off_list_answer_yields_empty_result() -> None: + output = _ClassifierOutput(labels=["Spaceship", "Boarding pass"]) + result = validate_labels(output, _opts("Invoice", "Receipt")) + assert result.labels == [] + + +# ── render_labels ──────────────────────────────────────────────────────────── + + +def test_render_labels_lists_the_vocabulary_names() -> None: + rendered = render_labels(_opts("Invoice", "Receipt")) + assert "Invoice" in rendered + assert "Receipt" in rendered + + +def test_render_labels_handles_empty_vocabulary() -> None: + assert "(none)" in render_labels([]) + + +# ── DocumentClassifierAgent (inline page text) ─────────────────────────────── + + +def _stub_model_answer(agent: DocumentClassifierAgent, labels: list[str]) -> AsyncMock: + mock = AsyncMock(return_value=SimpleNamespace(output=_ClassifierOutput(labels=labels))) + agent._agent.run = mock + return mock + + +@pytest.mark.anyio +async def test_classify_respects_request_supplied_vocabulary(runtime: AppRuntime) -> None: + agent = DocumentClassifierAgent(runtime) + run_mock = _stub_model_answer(agent, ["board minutes", "Invoice"]) + + result = await agent.classify( + ClassifyDocumentRequest( + file_name="minutes.pdf", + pages=[PageText(page_number=1, text="Minutes of the board meeting")], + labels=_opts("Board minutes", "Agenda"), + ) + ) + + # "Invoice" is off this request's vocabulary even though the default knows it. + assert result.labels == ["board-minutes"] + assert run_mock.await_args is not None + prompt = run_mock.await_args.args[0] + assert "Board minutes" in prompt + assert "Agenda" in prompt diff --git a/engine/tests/test_document_classifier_routes.py b/engine/tests/test_document_classifier_routes.py new file mode 100644 index 0000000000..76715d2862 --- /dev/null +++ b/engine/tests/test_document_classifier_routes.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from fastapi.testclient import TestClient + +from stirling.api import app +from stirling.api.dependencies import get_document_classifier_agent +from stirling.contracts import ( + ClassifyDocumentRequest, + ClassifyDocumentResponse, + DocumentClassificationResponse, +) + + +class StubClassifierAgent: + """Stands in for DocumentClassifierAgent so route tests don't call a model.""" + + def __init__(self, response: ClassifyDocumentResponse) -> None: + self._response = response + + async def classify(self, _request: ClassifyDocumentRequest) -> ClassifyDocumentResponse: + return self._response + + +@pytest.fixture +def classification_client() -> Iterator[TestClient]: + app.dependency_overrides[get_document_classifier_agent] = lambda: StubClassifierAgent( + # The result is label ids (the model's name answers, mapped to ids). + DocumentClassificationResponse(labels=["nda", "contract"]) + ) + try: + yield TestClient(app) + finally: + app.dependency_overrides.pop(get_document_classifier_agent, None) + + +_LABELS = [{"id": "nda", "name": "NDA"}, {"id": "contract", "name": "Contract"}] + + +def test_classify_returns_assigned_labels(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={ + "fileName": "nda.pdf", + "pages": [{"pageNumber": 1, "text": "Mutual NDA between A and B."}], + "labels": _LABELS, + }, + ) + assert response.status_code == 200 + assert response.json() == {"labels": ["nda", "contract"]} + + +def test_classify_accepts_allowed_labels_on_the_request(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={ + "fileName": "nda.pdf", + "pages": [], + "labels": [ + {"id": "contract", "name": "Contract"}, + {"id": "invoice", "name": "Invoice"}, + ], + }, + ) + assert response.status_code == 200 + + +def test_classify_accepts_empty_pages(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={"fileName": "blank.pdf", "pages": [], "labels": _LABELS}, + ) + assert response.status_code == 200 + + +def test_classify_rejects_empty_file_name(classification_client: TestClient) -> None: + response = classification_client.post( + "/api/v1/documents/classify", + json={"fileName": "", "pages": [], "labels": _LABELS}, + ) + assert response.status_code == 422 + + +def test_classify_rejects_missing_labels(classification_client: TestClient) -> None: + # The backend always supplies the vocabulary; a request without one is invalid. + response = classification_client.post( + "/api/v1/documents/classify", + json={"fileName": "nda.pdf", "pages": []}, + ) + assert response.status_code == 422 diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index 242c3885e3..acaaf00d80 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -3691,7 +3691,9 @@ workbench = "Workbench" [fileSidebar] addFiles = "Add files" +addingFiles = "Adding files…" collapse = "Collapse sidebar" +customizeGroups = "Customize groups" dropHint = "Open files to get started" dropToAdd = "Drop files to add" expand = "Expand sidebar" @@ -3704,8 +3706,11 @@ noFiles = "No files yet" openFileManager = "Browse all files & folders" openFromComputer = "Open from computer" openSettings = "Open settings" +other = "Other" +recent = "Recent" search = "Search" searchPlaceholder = "Search files..." +viewAll = "View all {{count}} files" [fileSidebar.fileItem] closeViewer = "Close viewer" @@ -3717,6 +3722,23 @@ updateOnServer = "Update on server" uploadToServer = "Upload to server" versionHistory = "Version history" +[fileSidebar.groupsModal] +add = "Add" +addLabel = "Add a label…" +categoryIconAria = "Choose an icon for {{name}}" +createCategory = "Create category" +delete = "Delete category" +done = "Done" +hide = "Hide category" +newCategory = "New category name…" +removeLabel = "Remove {{name}}" +rename = "Rename" +reset = "Reset to defaults" +search = "Search labels…" +show = "Show category" +subtitle = "Group your files into parent categories. Add existing or new labels to a category, rename it, or create your own. Files in none of your categories appear under “Other”." +title = "Sidebar categories" + [filesPage] addToWorkspace = "Add to workspace" addToWorkspaceCount = "Add {{count}} to workspace" @@ -3726,6 +3748,7 @@ backToFolder = "Back to {{folder}}" backToMyFiles = "Back to My Files" breadcrumbs = "Folder path" cancel = "Cancel" +classification = "Classification" clearSearch = "Clear search" clearSelection = "Clear selection" closeDetails = "Close details" @@ -3885,10 +3908,10 @@ uploadFilesFailedDetail = "Could not upload files: {{message}}" added = "Added" count = "Files" folder = "Folder" +labels = "Labels" modified = "Modified" name = "Name" size = "Size" -toolHistory = "Tool history" toolHistoryAtVersion = "Cumulative tool chain" totalSize = "Total size" type = "Type" @@ -5981,6 +6004,33 @@ print = "Enforcing before print" [policies.fields] selectedCount = "{{count}} selected" +[policies.labels] +add = "Add" +addPlaceholder = "Add a label…" +customNote = "Customized for your team." +defaultNote = "Using the built-in default, shared with your team." +duplicate = "\"{{name}}\" already exists." +edit = "Edit labels" +empty = "No labels yet." +export = "Export JSON" +iconAria = "Choose an icon for {{name}}" +import = "Import JSON" +importError = "Couldn't import that file." +managedNote = "Team labels are managed by your team leader." +modalSubtitle = "Shared with your whole team. The classifier picks the labels that fit each document." +modalTitle = "Classification labels" +pickIcon = "Use {{label}} icon" +removeAria = "Remove {{name}}" +resetToDefault = "Reset to default" +saveForTeam = "Save for team" +saving = "Saving…" +sectionLabel = "Classification labels" +startFromScratch = "Start from scratch" +teamCount = "team labels" +tooLong = "Labels can be at most {{max}} characters." +ungrouped = "Ungrouped" +view = "View labels" + [policies.pii] account = "Account numbers (labelled)" card = "Credit / debit cards" @@ -5992,17 +6042,30 @@ placeholder = "Select PII types" routing = "US routing numbers (ABA)" ssn = "Social Security numbers" +[policies.settings] +noneExport = "No policies currently run on export." +noneUpload = "No policies currently run on upload." +onExport = "On export" +onUpload = "On upload" +reorderHandle = "Drag to reorder" +runOrderDesc = "When more than one policy runs on the same trigger, they run in this order — each on the previous policy's output. Drag to reorder." +title = "Policy settings" + [policies.sidebar] activeCount = "{{count}} active" -infoAriaLabel = "What is a policy?" infoTooltip = "A policy is a fixed set of tools that runs automatically whenever it's triggered — for example when a new document arrives — enforcing rules like redacting PII with no manual steps." loading = "Loading…" +optionsAriaLabel = "Policy options" +policySettings = "Policy settings" railAriaLabel = "{{label}} policy — {{status}}" railSuffixActive = " (Active)" railSuffixPaused = " (Paused)" +retryFailed = "Retry failed policies ({{count}})" +rowProgress = "{{completed}} of {{total}} files processed" setUp = "Set up" title = "Policies" upgradeToEnterprise = "Upgrade to enterprise" +whatIsPolicy = "What is a policy?" [policies.status] active = "Active" diff --git a/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx b/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx index fe834b43d7..e183e45d6e 100644 --- a/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx +++ b/frontend/editor/src/core/components/fileEditor/FileEditorThumbnail.tsx @@ -295,6 +295,9 @@ const FileEditorThumbnail = ({ const [showVersionHistory, setShowVersionHistory] = useState(false); const policyEnforcing = policies.some((p) => p.enforcing); + // Accent of the policy currently enforcing, so the overlay's icon/spinner match + // that policy's badge instead of a fixed blue. + const enforcingAccent = policies.find((p) => p.enforcing)?.accentColor; const hoverActions = useMemo(() => { const uploadLabel = isUploaded @@ -536,7 +539,11 @@ const FileEditorThumbnail = ({ )} {/* Policy enforcement overlay — shown while any policy is in-flight */} - + {/* Thumbnail image or loading state */} (null); + const [classificationOpen, setClassificationOpen] = useState(false); + const classificationEnabled = useClassificationEnabled(); + const labelName = useLabelName(); // Version chain for the selected file; empty for v1 or multi-select. const [versionChain, setVersionChain] = useState([]); const singleFileForChain = files.length === 1 ? files[0] : null; @@ -103,6 +113,29 @@ export function FileDetailsPanel({ }; }, [singleFileForChain]); + // Show the file's classification labels: the stub's cached copy when present + // (free — no byte load), else read the PDF metadata via the shared service. + // Gated on classification being enabled (SaaS + AI): off-feature we never read + // the metadata or show the section, so a PDF that happens to carry the + // StirlingPDFClassification key never reveals the feature in a build without it. + useEffect(() => { + setClassification(null); + if (!classificationEnabled) return; + const stub = singleFileForChain; + if (!stub) return; + if (stub.classificationLabels && stub.classificationLabels.length > 0) { + setClassification(stub.classificationLabels); + return; + } + let cancelled = false; + void readStubClassificationLabels(stub).then((labels) => { + if (!cancelled && labels) setClassification(labels); + }); + return () => { + cancelled = true; + }; + }, [singleFileForChain, classificationEnabled]); + if (files.length === 0) { return null; } @@ -241,17 +274,56 @@ export function FileDetailsPanel({ /> )} - {single.toolHistory && single.toolHistory.length > 0 && ( -

-
- {t("filesPage.field.toolHistory", "Tool history")} -
- -
+ {classification && ( + <> + + {classificationOpen && ( +
+
+ + {t("filesPage.field.labels", "Labels")} + + + {classification.map((label) => ( + + {labelName(label)} + + ))} + +
+
+ )} + )} {/* Version journey. Each tool run writes a new StirlingFile with the same `originalFileId` and an incremented @@ -275,12 +347,41 @@ export function FileDetailsPanel({ )} ) : ( - + <> + + {versionsOpen && ( + + )} + ))} ) : ( diff --git a/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx b/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx index 0b6a75878b..47495fe61b 100644 --- a/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx +++ b/frontend/editor/src/core/components/filesPage/VersionTimeline.tsx @@ -57,6 +57,7 @@ export interface VersionTimelineProps { currentId: FileId; onAddToWorkspace: (fileIds: FileId[]) => void; onRemove: (fileIds: FileId[]) => void; + hideHeader?: boolean; } /** Version timeline with per-row tool deltas and collapse-when-long. */ @@ -65,6 +66,7 @@ export function VersionTimeline({ currentId, onAddToWorkspace, onRemove, + hideHeader = false, }: VersionTimelineProps) { const { t } = useTranslation(); const [expandedIds, setExpandedIds] = useState>(new Set()); @@ -122,15 +124,17 @@ export function VersionTimeline({ return (
-
- - {t("filesPage.field.versionHistory", "Version journey")} - - {t("filesPage.versionsCount", "{{count}} versions", { - count: ordered.length, - })} - -
+ {!hideHeader && ( +
+ + {t("filesPage.field.versionHistory", "Version journey")} + + {t("filesPage.versionsCount", "{{count}} versions", { + count: ordered.length, + })} + +
+ )}
    {rows.map((row, idx) => { const isLast = idx === rows.length - 1; diff --git a/frontend/editor/src/core/components/shared/FileDocIcon.tsx b/frontend/editor/src/core/components/shared/FileDocIcon.tsx index 135162c54d..aafaf9aabd 100644 --- a/frontend/editor/src/core/components/shared/FileDocIcon.tsx +++ b/frontend/editor/src/core/components/shared/FileDocIcon.tsx @@ -1,3 +1,5 @@ +import { accentColor } from "@app/utils/accentColors"; + export type FileDocVariant = | "pdf" | "spreadsheet" @@ -7,14 +9,15 @@ export type FileDocVariant = | "code" | "generic"; +// Theme-adaptive accent per file type (shared palette; see utils/accentColors.ts). export const VARIANT_COLORS: Record = { - pdf: "#DC2626", - spreadsheet: "#16a34a", - doc: "#2563eb", - image: "#7c3aed", - archive: "#ea580c", - code: "#0891b2", - generic: "#71717a", + pdf: accentColor("red"), + spreadsheet: accentColor("green"), + doc: accentColor("blue"), + image: accentColor("violet"), + archive: accentColor("orange"), + code: accentColor("cyan"), + generic: accentColor("gray"), }; export function FileDocIcon({ @@ -44,7 +47,7 @@ export function FileDocIcon({ const row1Y = 11, row2Y = 14; interior = ( - + @@ -56,7 +59,7 @@ export function FileDocIcon({ <>