Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fb28de4d5e | ||
|
|
51d3d27fd3 | ||
|
|
ccfd22b2a9 | ||
|
|
a5ee329c36 | ||
|
|
2091874050 | ||
|
|
22e8a82fa1 | ||
|
|
eb754d12c3 | ||
|
|
01751bf2f0 | ||
|
|
119eb1f5ad | ||
|
|
d3638d786d | ||
|
|
e5a258a648 | ||
|
|
c64369e56c | ||
|
|
f29500c138 | ||
|
|
9d11918bd8 | ||
|
|
8d2bb14f99 |
@@ -342,9 +342,70 @@ jobs:
|
||||
# Set port for output
|
||||
echo "v2_port=${V2_PORT}" >> $GITHUB_OUTPUT
|
||||
|
||||
# ---- Storybook preview (only when this PR touches stories/.storybook) ----
|
||||
# Runs inside the same approved-contributor-gated deploy job, so it deploys
|
||||
# under the exact same access rules as the app preview.
|
||||
- name: Detect Storybook changes
|
||||
id: sb-changes
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
storybook:
|
||||
- 'frontend/**/*.stories.@(ts|tsx|mdx)'
|
||||
- 'frontend/**/*.mdx'
|
||||
- 'frontend/.storybook/**'
|
||||
|
||||
- name: Set up Node.js for Storybook
|
||||
if: steps.sb-changes.outputs.storybook == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install Task for Storybook
|
||||
if: steps.sb-changes.outputs.storybook == 'true'
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Build and deploy Storybook
|
||||
id: storybook
|
||||
if: steps.sb-changes.outputs.storybook == 'true'
|
||||
env:
|
||||
VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
|
||||
VPS_USER: ${{ secrets.NEW_VPS_USERNAME }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# `prepare` generates the icon set stories import (not committed).
|
||||
task frontend:prepare
|
||||
task frontend:storybook:build
|
||||
PR=${{ needs.check-pr.outputs.pr_number }}
|
||||
# Served at the ROOT of its own port so Storybook's global MSW worker
|
||||
# (/mockServiceWorker.js) resolves. Port = PR + 20000 (bijective, offset
|
||||
# from the app preview's bare-PR-number port).
|
||||
SB_PORT=$((PR + 20000))
|
||||
DIR=/stirling/SB-PR-$PR
|
||||
tar czf storybook.tgz -C frontend/storybook-static .
|
||||
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
storybook.tgz "$VPS_USER@$VPS_HOST:/tmp/storybook-$PR.tgz"
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T \
|
||||
"$VPS_USER@$VPS_HOST" << ENDSSH
|
||||
set -e
|
||||
rm -rf "$DIR" && mkdir -p "$DIR"
|
||||
tar xzf /tmp/storybook-$PR.tgz -C "$DIR"
|
||||
rm -f /tmp/storybook-$PR.tgz
|
||||
docker rm -f storybook-pr-$PR 2>/dev/null || true
|
||||
docker run -d --name storybook-pr-$PR --restart unless-stopped \
|
||||
-p $SB_PORT:80 -v "$DIR":/usr/share/nginx/html:ro nginx:alpine
|
||||
ENDSSH
|
||||
echo "url=http://$VPS_HOST:$SB_PORT/" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Post V2 deployment URL to PR
|
||||
if: success()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SB_URL: ${{ steps.storybook.outputs.url }}
|
||||
SB_FILES: ${{ steps.sb-changes.outputs.storybook_files }}
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
@@ -376,11 +437,32 @@ jobs:
|
||||
? `🧩 **Admin portal** included - try it at [${deploymentUrl}/portal](${deploymentUrl}/portal).\n\n`
|
||||
: ``;
|
||||
|
||||
// Storybook preview: only present when this PR changed stories/config.
|
||||
const sbUrl = process.env.SB_URL;
|
||||
let storybookNote = "";
|
||||
if (sbUrl) {
|
||||
const files = JSON.parse(process.env.SB_FILES || "[]");
|
||||
const stories = files.filter((f) => /\.stories\.(ts|tsx|mdx)$/.test(f));
|
||||
const config = files.filter((f) => f.startsWith("frontend/.storybook/"));
|
||||
const shorten = (f) =>
|
||||
f.replace(/^frontend\/editor\/src\//, "").replace(/^frontend\//, "");
|
||||
const storyList = stories.map((f) => `- \`${shorten(f)}\``).join("\n");
|
||||
const configList = config.map((f) => `- \`${shorten(f)}\``).join("\n");
|
||||
const summary =
|
||||
`${stories.length} stor${stories.length === 1 ? "y" : "ies"} changed` +
|
||||
(config.length ? ` (+${config.length} config file${config.length === 1 ? "" : "s"})` : "");
|
||||
storybookNote =
|
||||
`📚 **Storybook:** [${sbUrl}](${sbUrl})\n\n` +
|
||||
`<details>\n<summary>${summary}</summary>\n\n` +
|
||||
(storyList ? `**Stories**\n${storyList}\n\n` : "") +
|
||||
(configList ? `**Config**\n${configList}\n` : "") +
|
||||
`</details>\n\n`;
|
||||
}
|
||||
|
||||
const commentBody = `## 🚀 V2 Auto-Deployment Complete!\n\n` +
|
||||
`Your V2 PR with embedded architecture has been deployed!\n\n` +
|
||||
`🔗 **Direct Test URL (non-SSL)** [${deploymentUrl}](${deploymentUrl})\n\n` +
|
||||
`🔐 **Secure HTTPS URL**: unsupported currently\n\n` +
|
||||
portalNote +
|
||||
storybookNote +
|
||||
`_This deployment will be automatically cleaned up when the PR is closed._\n\n` +
|
||||
`🔄 **Auto-deployed** for approved V2 contributors.`;
|
||||
|
||||
@@ -476,7 +558,11 @@ jobs:
|
||||
else
|
||||
echo "V2 PR directory not found, nothing to clean up"
|
||||
fi
|
||||
|
||||
|
||||
# Remove this PR's Storybook preview (container + files), if any.
|
||||
docker rm -f storybook-pr-${{ github.event.pull_request.number }} 2>/dev/null || true
|
||||
rm -rf /stirling/SB-PR-${{ github.event.pull_request.number }}
|
||||
|
||||
# Clean up old unused images (older than 2 weeks) but keep recent ones for reuse
|
||||
docker image prune -af --filter "until=336h" --filter "label!=keep=true" || true
|
||||
|
||||
|
||||
@@ -80,6 +80,12 @@ tasks:
|
||||
OPEN: '{{.OPEN | default ""}}'
|
||||
env:
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
# Dev-only browser-tab label so concurrent worktrees are distinguishable.
|
||||
# Only the worktree folder basename (e.g. "wt1") is exposed — never the
|
||||
# full path, hostname, or user. Consumed at dev-serve time by vite.config
|
||||
# and dropped from production builds.
|
||||
STIRLING_DEV_LABEL:
|
||||
sh: basename "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cmds:
|
||||
- npx vite editor --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}}
|
||||
|
||||
|
||||
@@ -453,6 +453,7 @@ The frontend is organized with a clear separation of concerns:
|
||||
|
||||
- **CRITICAL**: Always update translations in `en-US` only - all other languages (including `en-GB`) are handled separately
|
||||
- Translation files are located in `frontend/editor/public/locales/`
|
||||
- After changing any translation file, run `task pre-commit:fix`
|
||||
|
||||
## Important Notes
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.time.format.DateTimeFormatter;
|
||||
import java.util.Calendar;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -17,6 +18,9 @@ import stirling.software.common.model.PdfMetadata;
|
||||
@Service
|
||||
public class PdfMetadataService {
|
||||
|
||||
/** ({@code {labels}}). Written by the classify-and-label tool. */
|
||||
public static final String CLASSIFICATION_KEY = "StirlingPDFClassification";
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final String stirlingPDFLabel;
|
||||
private final UserServiceInterface userService;
|
||||
@@ -177,4 +181,14 @@ public class PdfMetadataService {
|
||||
}
|
||||
pdf.getDocumentInformation().setAuthor(author);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the document classifier's JSON result into the custom Info-dictionary field {@link
|
||||
* #CLASSIFICATION_KEY}, leaving all other metadata untouched.
|
||||
*/
|
||||
public void setClassificationMetadata(PDDocument pdf, String classificationJson) {
|
||||
PDDocumentInformation info = pdf.getDocumentInformation();
|
||||
info.setCustomMetadataValue(CLASSIFICATION_KEY, classificationJson);
|
||||
pdf.setDocumentInformation(info);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,12 +57,13 @@ public class RequestUriUtils {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Admin portal SPA shell. Served publicly like the editor root so a direct
|
||||
// nav / refresh to /portal loads the app (the JWT lives in localStorage, not
|
||||
// a cookie, so the server can't authenticate the navigation itself). The
|
||||
// Admin portal SPA shell (mounted at /processor — must match the frontend
|
||||
// PORTAL_BASENAME). Served publicly like the editor root so a direct nav /
|
||||
// refresh to /processor loads the app (the JWT lives in localStorage, not a
|
||||
// cookie, so the server can't authenticate the navigation itself). The
|
||||
// portal gates access via its own auth gate + RequirePortalAccess, and its
|
||||
// data APIs stay protected, so serving the shell pre-auth is safe.
|
||||
if (normalizedUri.equals("/portal") || normalizedUri.startsWith("/portal/")) {
|
||||
if (normalizedUri.equals("/processor") || normalizedUri.startsWith("/processor/")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -75,10 +75,10 @@ class RequestUriUtilsTest {
|
||||
|
||||
@Test
|
||||
void testIsStaticResource_portalShell() {
|
||||
// The admin portal SPA shell is served pre-auth so it's directly navigable.
|
||||
assertTrue(RequestUriUtils.isStaticResource("/portal"));
|
||||
assertTrue(RequestUriUtils.isStaticResource("/portal/users"));
|
||||
assertTrue(RequestUriUtils.isStaticResource("/app", "/app/portal"));
|
||||
// The admin portal SPA shell (/processor) is served pre-auth so it's directly navigable.
|
||||
assertTrue(RequestUriUtils.isStaticResource("/processor"));
|
||||
assertTrue(RequestUriUtils.isStaticResource("/processor/users"));
|
||||
assertTrue(RequestUriUtils.isStaticResource("/app", "/app/processor"));
|
||||
}
|
||||
|
||||
// --- isFrontendRoute tests ---
|
||||
|
||||
+29
@@ -305,6 +305,23 @@ public class GetInfoOnPDF {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Info-dictionary keys exposed above via typed getters; any other key in the dictionary is
|
||||
* surfaced as custom metadata (e.g. the classification policy's StirlingPDFClassification
|
||||
* entry).
|
||||
*/
|
||||
private static final java.util.Set<String> STANDARD_INFO_KEYS =
|
||||
java.util.Set.of(
|
||||
"Title",
|
||||
"Author",
|
||||
"Subject",
|
||||
"Keywords",
|
||||
"Producer",
|
||||
"Creator",
|
||||
"CreationDate",
|
||||
"ModDate",
|
||||
"Trapped");
|
||||
|
||||
private static ObjectNode extractMetadata(PDDocument document) {
|
||||
ObjectNode metadata = objectMapper.createObjectNode();
|
||||
|
||||
@@ -335,6 +352,18 @@ public class GetInfoOnPDF {
|
||||
if (modificationDate != null) {
|
||||
metadata.put("ModificationDate", modificationDate);
|
||||
}
|
||||
|
||||
// Surface custom Info-dictionary entries (anything beyond the
|
||||
// standard fields above) — e.g. StirlingPDFClassification
|
||||
for (String key : info.getMetadataKeys()) {
|
||||
if (STANDARD_INFO_KEYS.contains(key)) {
|
||||
continue;
|
||||
}
|
||||
String value = info.getCustomMetadataValue(key);
|
||||
if (value != null && !value.isBlank()) {
|
||||
metadata.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error extracting metadata: {}", e.getMessage());
|
||||
|
||||
+9
@@ -4,7 +4,9 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import stirling.software.proprietary.access.service.DefaultPrincipalResolver;
|
||||
import stirling.software.proprietary.access.service.DefaultTeamLeadLookup;
|
||||
import stirling.software.proprietary.access.service.PrincipalResolver;
|
||||
import stirling.software.proprietary.access.service.TeamLeadLookup;
|
||||
|
||||
/** Access-layer bean wiring. */
|
||||
@@ -17,4 +19,11 @@ public class AccessConfig {
|
||||
TeamLeadLookup defaultTeamLeadLookup() {
|
||||
return new DefaultTeamLeadLookup();
|
||||
}
|
||||
|
||||
/** USER/TEAM projection unless another bean is defined (e.g. the saas resolver). */
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(PrincipalResolver.class)
|
||||
PrincipalResolver defaultPrincipalResolver() {
|
||||
return new DefaultPrincipalResolver();
|
||||
}
|
||||
}
|
||||
|
||||
+33
-4
@@ -25,7 +25,9 @@ import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.model.ResourceGrant;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.security.database.repository.UserRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
|
||||
/** Admin endpoints to grant/revoke access to gated resources (the portal, integration configs). */
|
||||
@RestController
|
||||
@@ -36,6 +38,8 @@ import stirling.software.proprietary.security.model.User;
|
||||
public class ResourceGrantController {
|
||||
|
||||
private final ResourceAccessService accessService;
|
||||
private final UserRepository userRepository;
|
||||
private final TeamRepository teamRepository;
|
||||
|
||||
@GetMapping("/grants")
|
||||
public ResponseEntity<?> list(
|
||||
@@ -45,6 +49,14 @@ public class ResourceGrantController {
|
||||
return ResponseEntity.ok(grants.stream().map(this::toDto).toList());
|
||||
}
|
||||
|
||||
@GetMapping("/grants/by-principal")
|
||||
public ResponseEntity<?> listByPrincipal(
|
||||
@RequestParam PrincipalType principalType, @RequestParam Long principalId) {
|
||||
List<ResourceGrant> grants =
|
||||
accessService.listGrantsForPrincipal(principalType, principalId);
|
||||
return ResponseEntity.ok(grants.stream().map(this::toDto).toList());
|
||||
}
|
||||
|
||||
@PostMapping("/grants")
|
||||
public ResponseEntity<?> create(
|
||||
@RequestBody GrantRequest request, @AuthenticationPrincipal User admin) {
|
||||
@@ -57,17 +69,26 @@ public class ResourceGrantController {
|
||||
"error",
|
||||
"resourceType, principalType and principalId are required"));
|
||||
}
|
||||
// PORTAL is a singleton (empty resourceId); every other type must name a resource.
|
||||
boolean portal = request.resourceType() == ResourceType.PORTAL;
|
||||
if (!portal && (request.resourceId() == null || request.resourceId().isBlank())) {
|
||||
return ResponseEntity.badRequest()
|
||||
.body(Map.of("error", "resourceId is required for " + request.resourceType()));
|
||||
}
|
||||
Long principalId = request.principalId();
|
||||
String principalError = validatePrincipalExists(request.principalType(), principalId);
|
||||
if (principalError != null) {
|
||||
return ResponseEntity.badRequest().body(Map.of("error", principalError));
|
||||
}
|
||||
AccessPermission permission =
|
||||
request.permission() == null ? AccessPermission.USE : request.permission();
|
||||
// PORTAL is a singleton resource; its grants always target the whole type.
|
||||
String resourceId =
|
||||
request.resourceType() == ResourceType.PORTAL ? "" : request.resourceId();
|
||||
String resourceId = portal ? "" : request.resourceId();
|
||||
ResourceGrant grant =
|
||||
accessService.grant(
|
||||
request.resourceType(),
|
||||
resourceId,
|
||||
request.principalType(),
|
||||
request.principalId(),
|
||||
principalId,
|
||||
permission,
|
||||
admin);
|
||||
return ResponseEntity.ok(toDto(grant));
|
||||
@@ -79,6 +100,14 @@ public class ResourceGrantController {
|
||||
return ResponseEntity.ok(Map.of("message", "Grant revoked"));
|
||||
}
|
||||
|
||||
// Rejects grants to nonexistent principals (dead rows otherwise).
|
||||
private String validatePrincipalExists(PrincipalType type, Long id) {
|
||||
return switch (type) {
|
||||
case USER -> userRepository.existsById(id) ? null : "User " + id + " does not exist";
|
||||
case TEAM -> teamRepository.existsById(id) ? null : "Team " + id + " does not exist";
|
||||
};
|
||||
}
|
||||
|
||||
private Map<String, Object> toDto(ResourceGrant g) {
|
||||
Map<String, Object> m = new HashMap<>();
|
||||
m.put("id", g.getId());
|
||||
|
||||
+11
@@ -54,4 +54,15 @@ public abstract class OwnedResource {
|
||||
public Long getOwnerTeamId() {
|
||||
return ownerTeam != null ? ownerTeam.getId() : null;
|
||||
}
|
||||
|
||||
/** Owner as a principal ref; null when server-owned (admin-only ownership). */
|
||||
public PrincipalRef getOwnerRef() {
|
||||
if (getOwnerUserId() != null) {
|
||||
return PrincipalRef.user(getOwnerUserId());
|
||||
}
|
||||
if (getOwnerTeamId() != null) {
|
||||
return PrincipalRef.team(getOwnerTeamId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package stirling.software.proprietary.access.model;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/** A (type, id) principal pair; the atom grants and ownership are expressed in. */
|
||||
public record PrincipalRef(PrincipalType type, Long id) {
|
||||
|
||||
public static PrincipalRef user(Long id) {
|
||||
return new PrincipalRef(PrincipalType.USER, id);
|
||||
}
|
||||
|
||||
public static PrincipalRef team(Long id) {
|
||||
return new PrincipalRef(PrincipalType.TEAM, id);
|
||||
}
|
||||
|
||||
/** Canonical engine wire form, e.g. "user:12". */
|
||||
public String token() {
|
||||
return type.name().toLowerCase(Locale.ROOT) + ":" + id;
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package stirling.software.proprietary.access.model;
|
||||
|
||||
/** Who a {@link ResourceGrant} is granted to. Org-wide access is expressed via default policy. */
|
||||
/** Who a {@link ResourceGrant} is granted to. */
|
||||
public enum PrincipalType {
|
||||
USER,
|
||||
TEAM
|
||||
|
||||
+16
@@ -3,11 +3,15 @@ package stirling.software.proprietary.access.repository;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.model.ResourceGrant;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
@Repository
|
||||
public interface ResourceGrantRepository extends JpaRepository<ResourceGrant, Long> {
|
||||
@@ -18,8 +22,20 @@ public interface ResourceGrantRepository extends JpaRepository<ResourceGrant, Lo
|
||||
List<ResourceGrant> findByResourceTypeAndPrincipalTypeAndPrincipalId(
|
||||
ResourceType resourceType, PrincipalType principalType, Long principalId);
|
||||
|
||||
/** All grants held by a principal, across resource types (for the manage-access view). */
|
||||
List<ResourceGrant> findByPrincipalTypeAndPrincipalId(
|
||||
PrincipalType principalType, Long principalId);
|
||||
|
||||
void deleteByResourceTypeAndResourceId(ResourceType resourceType, String resourceId);
|
||||
|
||||
/** Removes every grant held by a principal; used when the user/team behind it is deleted. */
|
||||
void deleteByPrincipalTypeAndPrincipalId(PrincipalType principalType, Long principalId);
|
||||
|
||||
// Detach issued grants so deleting the granting user does not hit the FK.
|
||||
@Modifying
|
||||
@Query("update ResourceGrant g set g.grantedBy = null where g.grantedBy = :user")
|
||||
void clearGrantedBy(@Param("user") User user);
|
||||
|
||||
boolean existsByResourceTypeAndResourceIdAndPrincipalTypeAndPrincipalId(
|
||||
ResourceType resourceType,
|
||||
String resourceId,
|
||||
|
||||
+6
-1
@@ -11,7 +11,12 @@ import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/** {@code @PreAuthorize} bean for portal-access checks. Active in self-hosted and saas. */
|
||||
/**
|
||||
* {@code @PreAuthorize} bean for portal-access checks. Active in self-hosted and saas. Convention:
|
||||
* every portal-exclusive endpoint is gated with
|
||||
* {@code @PreAuthorize("@resourceAccess.canUsePortal()")}; endpoints shared with the editor (e.g.
|
||||
* the policies API) must NOT be.
|
||||
*/
|
||||
@Component("resourceAccess")
|
||||
@RequiredArgsConstructor
|
||||
public class ResourceAccessSecurity {
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package stirling.software.proprietary.access.service;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import stirling.software.proprietary.access.model.PrincipalRef;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
/**
|
||||
* Self-hosted projection: the user and their team. One deployment = one org, so ORG_ALL is open.
|
||||
*/
|
||||
public class DefaultPrincipalResolver implements PrincipalResolver {
|
||||
|
||||
@Override
|
||||
public Set<PrincipalRef> principalsOf(User user) {
|
||||
if (user == null) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<PrincipalRef> principals = new HashSet<>();
|
||||
principals.add(PrincipalRef.user(user.getId()));
|
||||
if (user.getTeam() != null) {
|
||||
principals.add(PrincipalRef.team(user.getTeam().getId()));
|
||||
}
|
||||
return principals;
|
||||
}
|
||||
|
||||
// Self-hosted is a single deployment-wide org, so ORG_ALL admits every authenticated user.
|
||||
@Override
|
||||
public boolean allowsDeploymentWideAccess() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package stirling.software.proprietary.access.service;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamMembershipRepository;
|
||||
|
||||
/** Real lookup backed by team_memberships LEADER rows; wins over the no-op default bean. */
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class MembershipTeamLeadLookup implements TeamLeadLookup {
|
||||
|
||||
private final TeamMembershipRepository memberships;
|
||||
|
||||
@Override
|
||||
public boolean isAnyTeamLeader(User user) {
|
||||
return user != null
|
||||
&& user.getId() != null
|
||||
&& memberships.existsByUserIdAndRole(user.getId(), TeamRole.LEADER);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLeaderOfTeam(User user, Long teamId) {
|
||||
return user != null
|
||||
&& user.getId() != null
|
||||
&& teamId != null
|
||||
&& memberships.existsByTeamIdAndUserIdAndRole(
|
||||
teamId, user.getId(), TeamRole.LEADER);
|
||||
}
|
||||
}
|
||||
+6
-2
@@ -36,15 +36,19 @@ public class OwnershipService {
|
||||
return accessService.canUseResource(
|
||||
type,
|
||||
String.valueOf(resource.getId()),
|
||||
resource.getOwnerUserId(),
|
||||
resource.getOwnerRef(),
|
||||
resource.getDefaultAccess(),
|
||||
user);
|
||||
}
|
||||
|
||||
/** Whether the user may manage the resource. */
|
||||
public boolean canManage(ResourceType type, OwnedResource resource, User user) {
|
||||
// Disabled resources bypass grants for MANAGE too: admin/owner only.
|
||||
if (!resource.isEnabled()) {
|
||||
return isAdmin(user) || isOwner(resource, user);
|
||||
}
|
||||
return accessService.canManageResource(
|
||||
type, String.valueOf(resource.getId()), resource.getOwnerUserId(), user);
|
||||
type, String.valueOf(resource.getId()), resource.getOwnerRef(), user);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package stirling.software.proprietary.access.service;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.proprietary.access.model.PrincipalRef;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
/** Projects a user onto the set of principals they act as. */
|
||||
public interface PrincipalResolver {
|
||||
|
||||
/** Every principal the user acts as; empty for a null user. */
|
||||
Set<PrincipalRef> principalsOf(User user);
|
||||
|
||||
/**
|
||||
* Whether this deployment treats every authenticated user as one org, so the {@code ORG_ALL}
|
||||
* default policy admits anyone. Self-hosted: true. Multi-tenant saas: false, so an {@code
|
||||
* ORG_ALL} resource can't leak across tenants. Defaults to false (deny) for safety.
|
||||
*/
|
||||
default boolean allowsDeploymentWideAccess() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Canonical wire tokens for the engine, e.g. "user:12". */
|
||||
default Set<String> principalTokens(User user) {
|
||||
return principalsOf(user).stream().map(PrincipalRef::token).collect(Collectors.toSet());
|
||||
}
|
||||
}
|
||||
+44
-27
@@ -14,6 +14,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.proprietary.access.model.AccessPermission;
|
||||
import stirling.software.proprietary.access.model.DefaultAccessPolicy;
|
||||
import stirling.software.proprietary.access.model.PrincipalRef;
|
||||
import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.model.ResourceGrant;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
@@ -29,6 +30,7 @@ public class ResourceAccessService {
|
||||
|
||||
private final ResourceGrantRepository grantRepository;
|
||||
private final TeamLeadLookup teamLeadLookup;
|
||||
private final PrincipalResolver principalResolver;
|
||||
|
||||
@Value("${security.portal.defaultAccess:ADMINS_AND_TEAM_LEADS}")
|
||||
private DefaultAccessPolicy portalDefaultPolicy;
|
||||
@@ -44,28 +46,28 @@ public class ResourceAccessService {
|
||||
public boolean canUseResource(
|
||||
ResourceType type,
|
||||
String resourceId,
|
||||
Long ownerUserId,
|
||||
PrincipalRef owner,
|
||||
DefaultAccessPolicy defaultPolicy,
|
||||
User user) {
|
||||
if (user == null) {
|
||||
return false;
|
||||
}
|
||||
if (isOwner(ownerUserId, user) || isAdmin(user)) {
|
||||
if (isOwner(owner, user) || isAdmin(user)) {
|
||||
return true;
|
||||
}
|
||||
if (hasGrant(type, normalize(resourceId), user, AccessPermission.USE)) {
|
||||
return true;
|
||||
}
|
||||
return matchesDefault(defaultPolicy, user);
|
||||
return matchesDefault(defaultPolicy, owner, user);
|
||||
}
|
||||
|
||||
/** Whether the user may manage (edit/delete/share) a resource. No default-policy fallback. */
|
||||
public boolean canManageResource(
|
||||
ResourceType type, String resourceId, Long ownerUserId, User user) {
|
||||
ResourceType type, String resourceId, PrincipalRef owner, User user) {
|
||||
if (user == null) {
|
||||
return false;
|
||||
}
|
||||
if (isOwner(ownerUserId, user) || isAdmin(user)) {
|
||||
if (isOwner(owner, user) || isAdmin(user)) {
|
||||
return true;
|
||||
}
|
||||
return hasGrant(type, normalize(resourceId), user, AccessPermission.MANAGE);
|
||||
@@ -110,21 +112,22 @@ public class ResourceAccessService {
|
||||
return grantRepository.findByResourceTypeAndResourceId(type, normalize(resourceId));
|
||||
}
|
||||
|
||||
/** Resource ids of the given type that this user (or their team) holds any grant on. */
|
||||
/** Every grant a principal holds, for the per-user/per-team manage-access view. */
|
||||
public List<ResourceGrant> listGrantsForPrincipal(
|
||||
PrincipalType principalType, Long principalId) {
|
||||
return grantRepository.findByPrincipalTypeAndPrincipalId(principalType, principalId);
|
||||
}
|
||||
|
||||
/** Resource ids of the given type that any of the user's principals holds a grant on. */
|
||||
public Set<String> grantedResourceIds(ResourceType type, User user) {
|
||||
if (user == null) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<String> ids = new HashSet<>();
|
||||
for (ResourceGrant g :
|
||||
grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId(
|
||||
type, PrincipalType.USER, user.getId())) {
|
||||
ids.add(g.getResourceId());
|
||||
}
|
||||
if (user.getTeam() != null) {
|
||||
for (PrincipalRef principal : principalResolver.principalsOf(user)) {
|
||||
for (ResourceGrant g :
|
||||
grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId(
|
||||
type, PrincipalType.TEAM, user.getTeam().getId())) {
|
||||
type, principal.type(), principal.id())) {
|
||||
ids.add(g.getResourceId());
|
||||
}
|
||||
}
|
||||
@@ -135,18 +138,12 @@ public class ResourceAccessService {
|
||||
|
||||
private boolean hasGrant(
|
||||
ResourceType type, String resourceId, User user, AccessPermission required) {
|
||||
Long teamId = user.getTeam() != null ? user.getTeam().getId() : null;
|
||||
Set<PrincipalRef> principals = principalResolver.principalsOf(user);
|
||||
for (ResourceGrant g : grantRepository.findByResourceTypeAndResourceId(type, resourceId)) {
|
||||
if (!permissionSatisfies(g.getPermission(), required)) {
|
||||
continue;
|
||||
}
|
||||
if (g.getPrincipalType() == PrincipalType.USER
|
||||
&& g.getPrincipalId().equals(user.getId())) {
|
||||
return true;
|
||||
}
|
||||
if (g.getPrincipalType() == PrincipalType.TEAM
|
||||
&& teamId != null
|
||||
&& g.getPrincipalId().equals(teamId)) {
|
||||
if (principals.contains(new PrincipalRef(g.getPrincipalType(), g.getPrincipalId()))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -161,20 +158,40 @@ public class ResourceAccessService {
|
||||
return held == AccessPermission.MANAGE;
|
||||
}
|
||||
|
||||
private boolean matchesDefault(DefaultAccessPolicy policy, User user) {
|
||||
private boolean matchesDefault(DefaultAccessPolicy policy, PrincipalRef owner, User user) {
|
||||
if (policy == null) {
|
||||
return false;
|
||||
}
|
||||
return switch (policy) {
|
||||
case ORG_ALL -> true;
|
||||
// Admins already pass above; only team leads here.
|
||||
case ADMINS_AND_TEAM_LEADS -> teamLeadLookup.isAnyTeamLeader(user);
|
||||
// Deployment-wide only where the resolver treats everyone as one org; saas resolvers
|
||||
// return false, so ORG_ALL cannot leak a tenant's resource to another tenant's users.
|
||||
case ORG_ALL -> principalResolver.allowsDeploymentWideAccess();
|
||||
// Admins already pass above; only team leads here, scoped to the owning team.
|
||||
case ADMINS_AND_TEAM_LEADS -> matchesTeamLeadDefault(owner, user);
|
||||
case EXPLICIT_ONLY -> false;
|
||||
};
|
||||
}
|
||||
|
||||
private boolean isOwner(Long ownerUserId, User user) {
|
||||
return ownerUserId != null && ownerUserId.equals(user.getId());
|
||||
// Portal (no owner) admits any team lead; a team-owned resource admits only that team's
|
||||
// leads; a user-owned resource admits no extra leads.
|
||||
private boolean matchesTeamLeadDefault(PrincipalRef owner, User user) {
|
||||
if (owner == null) {
|
||||
return teamLeadLookup.isAnyTeamLeader(user);
|
||||
}
|
||||
return owner.type() == PrincipalType.TEAM
|
||||
&& owner.id() != null
|
||||
&& teamLeadLookup.isLeaderOfTeam(user, owner.id());
|
||||
}
|
||||
|
||||
// Team owners are the owning team's leaders; plain members are not.
|
||||
private boolean isOwner(PrincipalRef owner, User user) {
|
||||
if (owner == null || owner.id() == null) {
|
||||
return false;
|
||||
}
|
||||
return switch (owner.type()) {
|
||||
case USER -> owner.id().equals(user.getId());
|
||||
case TEAM -> teamLeadLookup.isLeaderOfTeam(user, owner.id());
|
||||
};
|
||||
}
|
||||
|
||||
private boolean isAdmin(User user) {
|
||||
|
||||
+22
-5
@@ -18,15 +18,26 @@ public class SecretMasker {
|
||||
// Cap recursion so a pathologically nested payload cannot overflow the stack.
|
||||
private static final int MAX_DEPTH = 32;
|
||||
|
||||
// Key-name substrings that mark a value sensitive. Over-masking a non-secret is
|
||||
// safe; leaking a secret is not, so this errs broad - but a per-type schema
|
||||
// whitelist would be a stronger boundary for free-form config (follow-up).
|
||||
private static final Set<String> SENSITIVE_HINTS =
|
||||
Set.of(
|
||||
"secret",
|
||||
"password",
|
||||
"passphrase",
|
||||
"pwd",
|
||||
"token",
|
||||
"apikey",
|
||||
"accesskey",
|
||||
"credential",
|
||||
"privatekey");
|
||||
"privatekey",
|
||||
"authorization",
|
||||
"cookie",
|
||||
"session",
|
||||
"connectionstring",
|
||||
"bearer",
|
||||
"signature");
|
||||
|
||||
/** Replace sensitive values with the mask (recursively) for safe display. */
|
||||
public Map<String, Object> mask(Map<String, Object> config) {
|
||||
@@ -73,18 +84,24 @@ public class SecretMasker {
|
||||
|
||||
private Map<String, Object> merge(
|
||||
Map<String, Object> stored, Map<String, Object> incoming, int depth) {
|
||||
Map<String, Object> out = new LinkedHashMap<>(stored);
|
||||
// Replace semantics (PUT): the result is the incoming document, except a redacted secret
|
||||
// keeps its stored value. Keys absent from incoming are dropped, so edits can remove them.
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Object> e : incoming.entrySet()) {
|
||||
String key = e.getKey();
|
||||
Object value = e.getValue();
|
||||
if (isSensitive(key)) {
|
||||
if (!isRedacted(value, depth)) {
|
||||
if (isRedacted(value, depth)) {
|
||||
if (stored.containsKey(key)) {
|
||||
out.put(key, stored.get(key)); // keep the stored secret
|
||||
}
|
||||
} else {
|
||||
out.put(key, value); // a real new secret replaces the stored one
|
||||
}
|
||||
continue; // redacted (blank / mask) -> keep stored
|
||||
continue;
|
||||
}
|
||||
if (depth < MAX_DEPTH
|
||||
&& out.get(key) instanceof Map<?, ?> s
|
||||
&& stored.get(key) instanceof Map<?, ?> s
|
||||
&& value instanceof Map<?, ?> i) {
|
||||
out.put(key, merge(castMap(s), castMap(i), depth + 1));
|
||||
} else {
|
||||
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package stirling.software.proprietary.classification;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
import stirling.software.proprietary.classification.model.LabelsValidator;
|
||||
import stirling.software.proprietary.classification.store.ClassificationLabelStore;
|
||||
import stirling.software.proprietary.classification.store.TeamLabelsEntity;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
|
||||
/**
|
||||
* Read/write the team's classification label set — the flat vocabulary the document classifier runs
|
||||
* against. Shared and team-scoped exactly like policies: every user reads their own team's labels,
|
||||
* and only a user who may edit policies (a team leader on SaaS, the global admin self-hosted; see
|
||||
* {@link PolicyManagementAuthority}) may change it — gated only when login is enabled, since
|
||||
* single-user deployments trust the local operator. A team with no stored labels reads as {@code
|
||||
* 204}; that team has no vocabulary, so its documents are not classified (there is no built-in
|
||||
* default on the backend or the engine — the label data lives only in the frontend).
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/classification/labels")
|
||||
@Hidden
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Classification", description = "Team-scoped document-classification labels")
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class ClassificationLabelsController {
|
||||
|
||||
private final ClassificationLabelStore labelStore;
|
||||
private final PolicyManagementAuthority policyManagementAuthority;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final UserServiceInterface userService;
|
||||
|
||||
@GetMapping
|
||||
@Operation(
|
||||
summary = "Get the team's classification labels",
|
||||
description =
|
||||
"Returns the caller's team label set, or 204 when the team has none (its"
|
||||
+ " documents are then not classified).")
|
||||
public ResponseEntity<ClassificationLabels> getTeamLabels() {
|
||||
return labelStore
|
||||
.findByTeam(currentTeamId())
|
||||
.map(ResponseEntity::ok)
|
||||
.orElseGet(() -> ResponseEntity.noContent().build());
|
||||
}
|
||||
|
||||
@PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
summary = "Save the team's classification labels",
|
||||
description =
|
||||
"Validates and stores the label set for the caller's team, shared by everyone"
|
||||
+ " on the team. Requires the policy-editor role for the team.")
|
||||
public ResponseEntity<ClassificationLabels> saveTeamLabels(
|
||||
@RequestBody ClassificationLabels labels) {
|
||||
requireEditingAllowed();
|
||||
validate(labels);
|
||||
ClassificationLabels saved = labelStore.save(currentTeamId(), labels, currentUsername());
|
||||
return ResponseEntity.ok(saved);
|
||||
}
|
||||
|
||||
@DeleteMapping
|
||||
@Operation(
|
||||
summary = "Reset the team's classification labels",
|
||||
description =
|
||||
"Removes the team's stored label set; its documents are then not classified"
|
||||
+ " until labels are saved again. Requires the policy-editor role for the"
|
||||
+ " team.")
|
||||
public ResponseEntity<Void> resetTeamLabels() {
|
||||
requireEditingAllowed();
|
||||
labelStore.deleteByTeam(currentTeamId());
|
||||
return ResponseEntity.noContent().build();
|
||||
}
|
||||
|
||||
private static void validate(ClassificationLabels labels) {
|
||||
try {
|
||||
LabelsValidator.validate(labels);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Editing the team labels requires the editor role for the caller's team — the same gate
|
||||
* policies use (team leader on SaaS, global admin self-hosted). Single-user deployments (login
|
||||
* disabled) have no such role, so they trust the local operator.
|
||||
*/
|
||||
private void requireEditingAllowed() {
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()) {
|
||||
return;
|
||||
}
|
||||
if (!policyManagementAuthority.canEditPolicies()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.FORBIDDEN,
|
||||
"The team classification labels may only be changed by a team leader");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The caller's team key. With login disabled the single operator owns the {@link
|
||||
* TeamLabelsEntity#NO_TEAM} sentinel row; with login enabled a caller with no resolvable team
|
||||
* is an error rather than being dropped into the shared sentinel bucket (which would let
|
||||
* unteamed users read and overwrite each other's "team" labels).
|
||||
*/
|
||||
private Long currentTeamId() {
|
||||
Long teamId = policyManagementAuthority.currentUserTeamId();
|
||||
if (teamId != null) {
|
||||
return teamId;
|
||||
}
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()) {
|
||||
return TeamLabelsEntity.NO_TEAM;
|
||||
}
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.UNAUTHORIZED, "Could not resolve the current user's team");
|
||||
}
|
||||
|
||||
private String currentUsername() {
|
||||
return userService == null ? null : userService.getCurrentUsername();
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package stirling.software.proprietary.classification.model;
|
||||
|
||||
/**
|
||||
* One entry in the classification vocabulary. {@code id} is the label's stable identity (a slug,
|
||||
* unique within a set): it is what the engine returns and what is stored on the document. {@code
|
||||
* name} is the human display text the classifier model reasons over. {@code icon} is an optional
|
||||
* presentational key (a Material Symbols name shown in the file sidebar); the engine never sees it.
|
||||
*/
|
||||
public record ClassificationLabel(String id, String name, String icon) {}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.proprietary.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A flat multi-label classification vocabulary — the set of labels a document may be assigned.
|
||||
* Stored per team (admin-edited, shared by everyone on the team); the classifier runs against these
|
||||
* label names. A team with no stored set has no vocabulary, so its documents are not classified —
|
||||
* neither the backend nor the engine holds a default of its own.
|
||||
*/
|
||||
public record ClassificationLabels(List<ClassificationLabel> labels) {
|
||||
|
||||
public ClassificationLabels {
|
||||
labels = labels == null ? List.of() : List.copyOf(labels);
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package stirling.software.proprietary.classification.model;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Structural validation for a user- or admin-supplied label set, run before it is stored so a
|
||||
* malformed vocabulary can never reach the classifier. Mirrors the invariants the engine relies on:
|
||||
* non-blank ids and names, each unique within the set (ids exactly, names case-insensitively).
|
||||
*/
|
||||
public final class LabelsValidator {
|
||||
|
||||
private LabelsValidator() {}
|
||||
|
||||
// Generous upper bounds so a legitimate label set is never blocked, but a single team or user
|
||||
// can't store an unbounded blob that would bloat the row, balloon the classifier prompt, or
|
||||
// exhaust memory on deserialize.
|
||||
static final int MAX_LABELS = 500;
|
||||
static final int MAX_TEXT_LENGTH = 128;
|
||||
|
||||
// Icon is a Material Symbols key (lowercase, digits, hyphens). Enforce the SHAPE server-side —
|
||||
// the exact allowlist lives in the frontend — so a client bypassing the UI can't store
|
||||
// arbitrary
|
||||
// text that would render as garbage (or worse) in every teammate's sidebar.
|
||||
private static final Pattern ICON_KEY = Pattern.compile("^[a-z0-9-]+$");
|
||||
|
||||
/**
|
||||
* @throws IllegalArgumentException with a human-readable message when the label set is invalid.
|
||||
*/
|
||||
public static void validate(ClassificationLabels labels) {
|
||||
if (labels == null || labels.labels() == null) {
|
||||
throw new IllegalArgumentException("Labels are required");
|
||||
}
|
||||
if (labels.labels().size() > MAX_LABELS) {
|
||||
throw new IllegalArgumentException("Too many labels (max " + MAX_LABELS + ")");
|
||||
}
|
||||
Set<String> ids = new HashSet<>();
|
||||
Set<String> names = new HashSet<>();
|
||||
for (ClassificationLabel label : labels.labels()) {
|
||||
requireText(label.id(), "Label id");
|
||||
requireText(label.name(), "Label name");
|
||||
if (label.icon() != null && !label.icon().isEmpty()) {
|
||||
if (label.icon().length() > MAX_TEXT_LENGTH) {
|
||||
throw new IllegalArgumentException(
|
||||
"Label icon is too long (max " + MAX_TEXT_LENGTH + " characters)");
|
||||
}
|
||||
if (!ICON_KEY.matcher(label.icon()).matches()) {
|
||||
throw new IllegalArgumentException("Invalid label icon: " + label.icon());
|
||||
}
|
||||
}
|
||||
if (!ids.add(label.id().trim())) {
|
||||
throw new IllegalArgumentException("Duplicate label id: " + label.id());
|
||||
}
|
||||
if (!names.add(label.name().trim().toLowerCase(Locale.ROOT))) {
|
||||
throw new IllegalArgumentException("Duplicate label name: " + label.name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void requireText(String value, String field) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(field + " must not be blank");
|
||||
}
|
||||
if (value.trim().length() > MAX_TEXT_LENGTH) {
|
||||
throw new IllegalArgumentException(
|
||||
field + " is too long (max " + MAX_TEXT_LENGTH + " characters)");
|
||||
}
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
/**
|
||||
* Stores one {@link ClassificationLabels} set per team. A {@code null} teamId addresses the
|
||||
* unteamed set (login disabled / no resolvable team), mirroring how the policy store treats a null
|
||||
* team.
|
||||
*/
|
||||
public interface ClassificationLabelStore {
|
||||
|
||||
/** The team's stored labels, or empty when it has none (callers then skip classification). */
|
||||
Optional<ClassificationLabels> findByTeam(Long teamId);
|
||||
|
||||
/** Create or replace the team's labels. Returns the stored value. */
|
||||
ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy);
|
||||
|
||||
/** Remove the team's labels (reset to default). Returns whether a set existed. */
|
||||
boolean deleteByTeam(Long teamId);
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
/**
|
||||
* In-memory {@link ClassificationLabelStore} for tests and any future no-database mode. {@link
|
||||
* JpaClassificationLabelStore} is the runtime bean.
|
||||
*/
|
||||
public class InProcessClassificationLabelStore implements ClassificationLabelStore {
|
||||
|
||||
private final Map<Long, ClassificationLabels> byTeam = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public Optional<ClassificationLabels> findByTeam(Long teamId) {
|
||||
return Optional.ofNullable(byTeam.get(key(teamId)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) {
|
||||
byTeam.put(key(teamId), labels);
|
||||
return labels;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteByTeam(Long teamId) {
|
||||
return byTeam.remove(key(teamId)) != null;
|
||||
}
|
||||
|
||||
private static long key(Long teamId) {
|
||||
return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId;
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Durable {@link ClassificationLabelStore} backed by JPA; the runtime store. Gated on {@code
|
||||
* policies.enabled} — stored labels only matter when the Classification policy can run — so it
|
||||
* shares the policy subsystem's on/off switch. Each label set is persisted as JSON via {@link
|
||||
* TeamLabelsEntity}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class JpaClassificationLabelStore implements ClassificationLabelStore {
|
||||
|
||||
private final TeamLabelsRepository teamRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public Optional<ClassificationLabels> findByTeam(Long teamId) {
|
||||
return teamRepository
|
||||
.findById(key(teamId))
|
||||
.flatMap(entity -> parse(entity.getLabelsJson(), "team " + teamId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) {
|
||||
TeamLabelsEntity entity = new TeamLabelsEntity();
|
||||
entity.setTeamId(key(teamId));
|
||||
entity.setLabelsJson(objectMapper.writeValueAsString(labels));
|
||||
entity.setUpdatedAt(Instant.now());
|
||||
entity.setUpdatedBy(updatedBy);
|
||||
teamRepository.save(entity);
|
||||
return labels;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteByTeam(Long teamId) {
|
||||
long id = key(teamId);
|
||||
if (!teamRepository.existsById(id)) {
|
||||
return false;
|
||||
}
|
||||
teamRepository.deleteById(id);
|
||||
return true;
|
||||
}
|
||||
|
||||
private Optional<ClassificationLabels> parse(String json, String owner) {
|
||||
try {
|
||||
return Optional.of(objectMapper.readValue(json, ClassificationLabels.class));
|
||||
} catch (JacksonException e) {
|
||||
// A stored label set that no longer parses (corruption / manual DB edit) must not break
|
||||
// classification: drop it so the caller treats the team as having no labels (and skips
|
||||
// classification) rather than surfacing a 500 on every upload.
|
||||
log.warn("Discarding unparseable stored labels for {}: {}", owner, e.getMessage());
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
/** Map the nullable team id onto the entity's non-null key (sentinel for the unteamed case). */
|
||||
private static long key(Long teamId) {
|
||||
return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId;
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Table;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* JPA row for a team's classification labels — one row per team. The label set lives as JSON in
|
||||
* {@code labelsJson} (authoritative on read). {@code teamId} is the natural key; the sentinel
|
||||
* {@link #NO_TEAM} stands in for the unteamed (login-disabled / self-hosted single-team) case,
|
||||
* since a primary key can't be null (policies store a nullable {@code team_id}, but this table is
|
||||
* keyed one-per-team). Kept decoupled from the security entities — {@code teamId} is a plain value,
|
||||
* not a foreign key — so classification can be enabled or disabled without touching them.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "classification_labels")
|
||||
@NoArgsConstructor
|
||||
@Getter
|
||||
@Setter
|
||||
public class TeamLabelsEntity implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/** Sentinel key for the unteamed label set (login disabled / no resolvable team). */
|
||||
public static final long NO_TEAM = 0L;
|
||||
|
||||
@Id
|
||||
@Column(name = "team_id")
|
||||
private long teamId;
|
||||
|
||||
@Column(name = "labels_json", columnDefinition = "text")
|
||||
private String labelsJson;
|
||||
|
||||
@Column(name = "updated_at")
|
||||
private Instant updatedAt;
|
||||
|
||||
@Column(name = "updated_by")
|
||||
private String updatedBy;
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface TeamLabelsRepository extends JpaRepository<TeamLabelsEntity, Long> {}
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
package stirling.software.proprietary.controller.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.service.PdfMetadataService;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.store.ClassificationLabelStore;
|
||||
import stirling.software.proprietary.model.api.ai.AiPageText;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.service.AiEngineClient;
|
||||
import stirling.software.proprietary.service.PdfContentExtractor;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Dispatchable tool that classifies a PDF and writes the result into its metadata.
|
||||
*
|
||||
* <p>Runs as a Classification-policy pipeline step: it reads a bounded page window, asks the AI
|
||||
* engine to classify the document against the caller's team label set, and stores the engine's JSON
|
||||
* answer — minus the transport-only {@code outcome} field — in the custom Info-dictionary key
|
||||
* {@link PdfMetadataService#CLASSIFICATION_KEY}. Returns the labelled PDF. Not intended for direct
|
||||
* client use.
|
||||
*/
|
||||
@Slf4j
|
||||
@Hidden
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/ai/tools")
|
||||
@Tag(name = "AI Tools", description = "Dispatchable AI-backed tools.")
|
||||
public class ClassifyLabelController {
|
||||
|
||||
/** Pages read from each end of the document — mirrors the engine's window. */
|
||||
private static final int WINDOW_PAGES = 2;
|
||||
|
||||
private static final String CLASSIFY_ENDPOINT = "/api/v1/documents/classify";
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final PdfContentExtractor pdfContentExtractor;
|
||||
private final PdfMetadataService pdfMetadataService;
|
||||
private final AiEngineClient aiEngineClient;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final UserServiceInterface userService;
|
||||
|
||||
/**
|
||||
* Present only when the policy subsystem is enabled ({@code policies.enabled}); the store and
|
||||
* team authority are gated on it. Null otherwise, in which case there are no team labels to
|
||||
* classify against and the document is passed through unlabelled.
|
||||
*/
|
||||
private final ClassificationLabelStore labelStore;
|
||||
|
||||
private final PolicyManagementAuthority policyManagementAuthority;
|
||||
|
||||
public ClassifyLabelController(
|
||||
CustomPDFDocumentFactory pdfDocumentFactory,
|
||||
TempFileManager tempFileManager,
|
||||
PdfContentExtractor pdfContentExtractor,
|
||||
PdfMetadataService pdfMetadataService,
|
||||
AiEngineClient aiEngineClient,
|
||||
ObjectMapper objectMapper,
|
||||
@Autowired(required = false) UserServiceInterface userService,
|
||||
@Autowired(required = false) ClassificationLabelStore labelStore,
|
||||
@Autowired(required = false) PolicyManagementAuthority policyManagementAuthority) {
|
||||
this.pdfDocumentFactory = pdfDocumentFactory;
|
||||
this.tempFileManager = tempFileManager;
|
||||
this.pdfContentExtractor = pdfContentExtractor;
|
||||
this.pdfMetadataService = pdfMetadataService;
|
||||
this.aiEngineClient = aiEngineClient;
|
||||
this.objectMapper = objectMapper;
|
||||
this.userService = userService;
|
||||
this.labelStore = labelStore;
|
||||
this.policyManagementAuthority = policyManagementAuthority;
|
||||
}
|
||||
|
||||
@PostMapping(value = "/classify-and-label", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Classify a PDF and label its metadata",
|
||||
description =
|
||||
"Reads the first two and last two pages, classifies the document via the AI"
|
||||
+ " engine, and stores the result in the StirlingPDFClassification"
|
||||
+ " metadata field. Dispatched by the Classification policy; not"
|
||||
+ " intended for direct client use.")
|
||||
public ResponseEntity<Resource> classifyAndLabel(
|
||||
@RequestParam("fileInput") MultipartFile fileInput) throws IOException {
|
||||
try (PDDocument document = pdfDocumentFactory.load(fileInput, true)) {
|
||||
String fileName = safeFileName(fileInput.getOriginalFilename());
|
||||
|
||||
List<EngineLabel> allowed = resolveAllowedLabels();
|
||||
if (allowed.isEmpty()) {
|
||||
// No vocabulary to classify against (the team stored no labels): pass the file
|
||||
// through unlabelled rather than ask the engine to classify against nothing.
|
||||
log.debug("[classify-and-label] {} has no team labels; skipping", fileName);
|
||||
return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager);
|
||||
}
|
||||
|
||||
List<AiPageText> pages = extractWindow(document);
|
||||
String requestBody =
|
||||
objectMapper.writeValueAsString(
|
||||
new ClassifyEngineRequest(fileName, pages, allowed));
|
||||
|
||||
String userId = userService != null ? userService.getCurrentUsername() : null;
|
||||
String responseJson = aiEngineClient.post(CLASSIFY_ENDPOINT, requestBody, userId);
|
||||
|
||||
pdfMetadataService.setClassificationMetadata(document, toMetadataValue(responseJson));
|
||||
log.debug("[classify-and-label] labelled {} ({} window pages)", fileName, pages.size());
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager);
|
||||
}
|
||||
}
|
||||
|
||||
private List<AiPageText> extractWindow(PDDocument document) throws IOException {
|
||||
List<AiPageText> pages = new ArrayList<>();
|
||||
for (int pageNumber : windowPageNumbers(document.getNumberOfPages(), WINDOW_PAGES)) {
|
||||
String text = pdfContentExtractor.extractPageTextRaw(document, pageNumber);
|
||||
if (text != null && !text.isBlank()) {
|
||||
pages.add(new AiPageText(pageNumber, text));
|
||||
}
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
/** First and last {@code window} page numbers (1-based), de-duplicated and in order. */
|
||||
static List<Integer> windowPageNumbers(int pageCount, int window) {
|
||||
Set<Integer> numbers = new LinkedHashSet<>();
|
||||
for (int page = 1; page <= Math.min(window, pageCount); page++) {
|
||||
numbers.add(page);
|
||||
}
|
||||
for (int page = Math.max(1, pageCount - window + 1); page <= pageCount; page++) {
|
||||
numbers.add(page);
|
||||
}
|
||||
return new ArrayList<>(numbers);
|
||||
}
|
||||
|
||||
/** Drop the transport-only {@code outcome} discriminator; keep the rest verbatim. */
|
||||
private String toMetadataValue(String engineResponseJson) {
|
||||
JsonNode node = objectMapper.readTree(engineResponseJson);
|
||||
if (node instanceof ObjectNode object) {
|
||||
object.remove("outcome");
|
||||
}
|
||||
return objectMapper.writeValueAsString(node);
|
||||
}
|
||||
|
||||
private static String safeFileName(String originalFilename) {
|
||||
String name = Filenames.toSimpleFileName(originalFilename);
|
||||
return (name == null || name.isBlank()) ? "classified.pdf" : name;
|
||||
}
|
||||
|
||||
/**
|
||||
* The allowed labels for the caller's team as {@code {id, name}} pairs, de-duplicated by id.
|
||||
* The engine shows the model the names and returns the ids (icons are presentational and never
|
||||
* sent). Returns an empty list — the caller then skips classification — when the policy
|
||||
* subsystem is disabled (no store) or the team has no stored labels. The engine holds no
|
||||
* default vocabulary of its own, so a team's stored labels are the only source.
|
||||
*/
|
||||
private List<EngineLabel> resolveAllowedLabels() {
|
||||
if (labelStore == null) {
|
||||
return List.of();
|
||||
}
|
||||
Long teamId =
|
||||
policyManagementAuthority == null
|
||||
? null
|
||||
: policyManagementAuthority.currentUserTeamId();
|
||||
|
||||
Map<String, EngineLabel> byId = new LinkedHashMap<>();
|
||||
labelStore.findByTeam(teamId).ifPresent(labels -> collectLabels(labels.labels(), byId));
|
||||
|
||||
return List.copyOf(byId.values());
|
||||
}
|
||||
|
||||
private static void collectLabels(
|
||||
List<ClassificationLabel> labels, Map<String, EngineLabel> into) {
|
||||
for (ClassificationLabel label : labels) {
|
||||
if (label.id() == null
|
||||
|| label.id().isBlank()
|
||||
|| label.name() == null
|
||||
|| label.name().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
into.putIfAbsent(label.id(), new EngineLabel(label.id(), label.name()));
|
||||
}
|
||||
}
|
||||
|
||||
/** One allowed label sent to the engine: stable id + the name the model reasons over. */
|
||||
private record EngineLabel(String id, String name) {}
|
||||
|
||||
/** Request body for the engine's {@code /api/v1/documents/classify} endpoint. */
|
||||
private record ClassifyEngineRequest(
|
||||
String fileName, List<AiPageText> pages, List<EngineLabel> labels) {}
|
||||
}
|
||||
+50
-3
@@ -5,6 +5,7 @@ import static stirling.software.common.util.ProviderUtils.validateProvider;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -28,13 +29,16 @@ import stirling.software.common.model.ApplicationProperties.Security.OAUTH2.Clie
|
||||
import stirling.software.common.model.ApplicationProperties.Security.SAML2;
|
||||
import stirling.software.common.model.FileInfo;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.common.model.oauth2.GitHubProvider;
|
||||
import stirling.software.common.model.oauth2.GoogleProvider;
|
||||
import stirling.software.common.model.oauth2.KeycloakProvider;
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.audit.AuditEventType;
|
||||
import stirling.software.proprietary.audit.AuditLevel;
|
||||
import stirling.software.proprietary.config.AuditConfigurationProperties;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.model.TeamMembership;
|
||||
import stirling.software.proprietary.model.dto.TeamWithUserCountDTO;
|
||||
import stirling.software.proprietary.repository.PersistentAuditEventRepository;
|
||||
import stirling.software.proprietary.security.config.EnterpriseEndpoint;
|
||||
@@ -44,6 +48,7 @@ import stirling.software.proprietary.security.model.Authority;
|
||||
import stirling.software.proprietary.security.model.SessionEntity;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.model.dto.AdminUserSummary;
|
||||
import stirling.software.proprietary.security.repository.TeamMembershipRepository;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
import stirling.software.proprietary.security.saml2.CustomSaml2AuthenticatedPrincipal;
|
||||
import stirling.software.proprietary.security.service.DatabaseServiceInterface;
|
||||
@@ -65,6 +70,7 @@ public class ProprietaryUIDataController {
|
||||
private final SessionPersistentRegistry sessionPersistentRegistry;
|
||||
private final UserRepository userRepository;
|
||||
private final TeamRepository teamRepository;
|
||||
private final TeamMembershipRepository teamMembershipRepository;
|
||||
private final SessionRepository sessionRepository;
|
||||
private final DatabaseServiceInterface databaseService;
|
||||
private final boolean runningEE;
|
||||
@@ -73,6 +79,7 @@ public class ProprietaryUIDataController {
|
||||
private final PersistentAuditEventRepository auditRepository;
|
||||
private final MfaService mfaService;
|
||||
private final LoginAttemptService loginAttemptService;
|
||||
private final ResourceAccessService resourceAccessService;
|
||||
|
||||
public ProprietaryUIDataController(
|
||||
ApplicationProperties applicationProperties,
|
||||
@@ -80,6 +87,7 @@ public class ProprietaryUIDataController {
|
||||
SessionPersistentRegistry sessionPersistentRegistry,
|
||||
UserRepository userRepository,
|
||||
TeamRepository teamRepository,
|
||||
TeamMembershipRepository teamMembershipRepository,
|
||||
SessionRepository sessionRepository,
|
||||
DatabaseServiceInterface databaseService,
|
||||
ObjectMapper objectMapper,
|
||||
@@ -87,12 +95,14 @@ public class ProprietaryUIDataController {
|
||||
UserLicenseSettingsService licenseSettingsService,
|
||||
PersistentAuditEventRepository auditRepository,
|
||||
MfaService mfaService,
|
||||
LoginAttemptService loginAttemptService) {
|
||||
LoginAttemptService loginAttemptService,
|
||||
ResourceAccessService resourceAccessService) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
this.auditConfig = auditConfig;
|
||||
this.sessionPersistentRegistry = sessionPersistentRegistry;
|
||||
this.userRepository = userRepository;
|
||||
this.teamRepository = teamRepository;
|
||||
this.teamMembershipRepository = teamMembershipRepository;
|
||||
this.sessionRepository = sessionRepository;
|
||||
this.databaseService = databaseService;
|
||||
this.objectMapper = objectMapper;
|
||||
@@ -101,6 +111,7 @@ public class ProprietaryUIDataController {
|
||||
this.auditRepository = auditRepository;
|
||||
this.mfaService = mfaService;
|
||||
this.loginAttemptService = loginAttemptService;
|
||||
this.resourceAccessService = resourceAccessService;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -370,8 +381,11 @@ public class ProprietaryUIDataController {
|
||||
boolean premiumEnabled = applicationProperties.getPremium().isEnabled();
|
||||
|
||||
// Convert User entities to AdminUserSummary DTOs to exclude sensitive fields
|
||||
Set<Long> leaderUserIds = leaderUserIds();
|
||||
List<AdminUserSummary> userSummaries =
|
||||
sortedUsers.stream().map(this::convertUserToSummary).toList();
|
||||
sortedUsers.stream()
|
||||
.map(user -> convertUserToSummary(user, leaderUserIds))
|
||||
.toList();
|
||||
|
||||
AdminSettingsData data = new AdminSettingsData();
|
||||
data.setUsers(userSummaries);
|
||||
@@ -390,6 +404,10 @@ public class ProprietaryUIDataController {
|
||||
data.setLicenseMaxUsers(licenseMaxUsers);
|
||||
data.setPremiumEnabled(premiumEnabled);
|
||||
data.setMailEnabled(applicationProperties.getMail().isEnabled());
|
||||
// Email invites need the invites toggle AND SMTP on; matches the inviteUsers precondition.
|
||||
data.setEmailInvitesEnabled(
|
||||
applicationProperties.getMail().isEnableInvites()
|
||||
&& applicationProperties.getMail().isEnabled());
|
||||
data.setUserSettings(userSettings);
|
||||
data.setLockedUsers(loginAttemptService.getAllBlockedUsers());
|
||||
|
||||
@@ -468,9 +486,18 @@ public class ProprietaryUIDataController {
|
||||
teamLastRequest.put(teamId, lastActivity);
|
||||
}
|
||||
|
||||
Map<Long, List<String>> teamOwners = new HashMap<>();
|
||||
for (TeamMembership row :
|
||||
teamMembershipRepository.findByRoleFetchingUserAndTeam(TeamRole.LEADER)) {
|
||||
teamOwners
|
||||
.computeIfAbsent(row.getTeam().getId(), id -> new ArrayList<>())
|
||||
.add(row.getUser().getUsername());
|
||||
}
|
||||
|
||||
TeamsData data = new TeamsData();
|
||||
data.setTeamsWithCounts(teamsWithCounts);
|
||||
data.setTeamLastRequest(teamLastRequest);
|
||||
data.setTeamOwners(teamOwners);
|
||||
|
||||
return ResponseEntity.ok(data);
|
||||
}
|
||||
@@ -510,11 +537,17 @@ public class ProprietaryUIDataController {
|
||||
userLastRequest.put(username, lastRequest);
|
||||
}
|
||||
|
||||
Set<Long> ownerUserIds =
|
||||
teamMembershipRepository.findByTeamIdAndRole(id, TeamRole.LEADER).stream()
|
||||
.map(row -> row.getUser().getId())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
TeamDetailsData data = new TeamDetailsData();
|
||||
data.setTeam(team);
|
||||
data.setTeamUsers(teamUsers);
|
||||
data.setAvailableUsers(availableUsers);
|
||||
data.setUserLastRequest(userLastRequest);
|
||||
data.setOwnerUserIds(ownerUserIds);
|
||||
|
||||
return ResponseEntity.ok(data);
|
||||
}
|
||||
@@ -535,13 +568,24 @@ public class ProprietaryUIDataController {
|
||||
return ResponseEntity.ok(data);
|
||||
}
|
||||
|
||||
/** User ids holding a LEADER membership on any team. */
|
||||
private Set<Long> leaderUserIds() {
|
||||
return teamMembershipRepository.findByRoleFetchingUserAndTeam(TeamRole.LEADER).stream()
|
||||
.map(row -> row.getUser().getId())
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert User entity to AdminUserSummary DTO, excluding sensitive fields like password and
|
||||
* apiKey.
|
||||
*/
|
||||
private AdminUserSummary convertUserToSummary(User user) {
|
||||
private AdminUserSummary convertUserToSummary(User user, Set<Long> leaderUserIds) {
|
||||
AdminUserSummary summary = new AdminUserSummary();
|
||||
summary.setId(user.getId());
|
||||
summary.setTeamLead(leaderUserIds.contains(user.getId()));
|
||||
// Authoritative portal access, same call /me uses, so the roster honors the configured
|
||||
// policy instead of the frontend guessing from role/team-leadership.
|
||||
summary.setPortalAccess(resourceAccessService.canAccessPortal(user));
|
||||
summary.setUsername(user.getUsername());
|
||||
summary.setEmail(user.getUsername()); // Use username as email for consistency
|
||||
summary.setRoleName(user.getRoleName());
|
||||
@@ -609,6 +653,7 @@ public class ProprietaryUIDataController {
|
||||
private int licenseMaxUsers;
|
||||
private boolean premiumEnabled;
|
||||
private boolean mailEnabled;
|
||||
private boolean emailInvitesEnabled;
|
||||
private Map<String, Map<String, String>> userSettings;
|
||||
private List<String> lockedUsers;
|
||||
}
|
||||
@@ -629,6 +674,7 @@ public class ProprietaryUIDataController {
|
||||
public static class TeamsData {
|
||||
private List<TeamWithUserCountDTO> teamsWithCounts;
|
||||
private Map<Long, Date> teamLastRequest;
|
||||
private Map<Long, List<String>> teamOwners;
|
||||
}
|
||||
|
||||
@Data
|
||||
@@ -637,6 +683,7 @@ public class ProprietaryUIDataController {
|
||||
private List<User> teamUsers;
|
||||
private List<User> availableUsers;
|
||||
private Map<String, Date> userLastRequest;
|
||||
private Set<Long> ownerUserIds;
|
||||
}
|
||||
|
||||
@Data
|
||||
|
||||
+3
-1
@@ -29,7 +29,9 @@ import stirling.software.proprietary.security.model.User;
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/integrations")
|
||||
@RequiredArgsConstructor
|
||||
@PreAuthorize("isAuthenticated()")
|
||||
// Portal-exclusive: server-side portal-access boundary, not just isAuthenticated. Per-config
|
||||
// ownership is still enforced in the service layer.
|
||||
@PreAuthorize("@resourceAccess.canUsePortal()")
|
||||
@Tag(name = "Integrations", description = "Manage S3/MCP/API integration configurations")
|
||||
public class IntegrationConfigController {
|
||||
|
||||
|
||||
+24
-1
@@ -1,12 +1,16 @@
|
||||
package stirling.software.proprietary.integration.crypto;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Arrays;
|
||||
import java.util.Base64;
|
||||
import java.util.EnumSet;
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.KeyGenerator;
|
||||
@@ -74,7 +78,7 @@ public class CredentialEncryption {
|
||||
generator.init(256);
|
||||
SecretKey generated = generator.generateKey();
|
||||
Files.createDirectories(path.getParent());
|
||||
Files.writeString(path, Base64.getEncoder().encodeToString(generated.getEncoded()));
|
||||
writeOwnerOnly(path, Base64.getEncoder().encodeToString(generated.getEncoded()));
|
||||
log.warn(
|
||||
"Generated a new credential encryption key at {}. Back this file up: losing it"
|
||||
+ " makes stored integration secrets unrecoverable.",
|
||||
@@ -85,6 +89,25 @@ public class CredentialEncryption {
|
||||
}
|
||||
}
|
||||
|
||||
// The master key decrypts every stored integration secret, so create it 0600
|
||||
// (owner-only) atomically. On non-POSIX filesystems (Windows) the config-dir
|
||||
// ACL is the protection; we still create the file, just without POSIX perms.
|
||||
private static void writeOwnerOnly(Path path, String content) throws IOException {
|
||||
EnumSet<PosixFilePermission> ownerOnly =
|
||||
EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE);
|
||||
try {
|
||||
Files.createFile(path, PosixFilePermissions.asFileAttribute(ownerOnly));
|
||||
} catch (UnsupportedOperationException e) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Files.writeString(path, content);
|
||||
try {
|
||||
Files.setPosixFilePermissions(path, ownerOnly);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
// Non-POSIX filesystem: nothing to tighten here.
|
||||
}
|
||||
}
|
||||
|
||||
public static String encrypt(String plaintext) {
|
||||
if (plaintext == null) {
|
||||
return null;
|
||||
|
||||
+9
@@ -18,4 +18,13 @@ public interface IntegrationConfigRepository extends JpaRepository<IntegrationCo
|
||||
List<IntegrationConfig> findByOwnerTeam(Team ownerTeam);
|
||||
|
||||
List<IntegrationConfig> findByScope(OwnerScope scope);
|
||||
|
||||
// Nested path: OwnedResource has a getOwnerTeamId() convenience getter but no such persistent
|
||||
// attribute, so the plain "...OwnerTeamId" derivation resolves to a phantom property and throws
|
||||
// UnknownPathException. The underscore forces the real ownerTeam.id association path.
|
||||
boolean existsByOwnerTeam_Id(Long teamId);
|
||||
|
||||
void deleteByOwnerUser(User ownerUser);
|
||||
|
||||
void deleteByOwnerTeam_Id(Long teamId);
|
||||
}
|
||||
|
||||
+11
@@ -16,6 +16,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.proprietary.access.model.DefaultAccessPolicy;
|
||||
import stirling.software.proprietary.access.model.OwnerScope;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.access.repository.ResourceGrantRepository;
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.access.service.SecretMasker;
|
||||
import stirling.software.proprietary.integration.dto.IntegrationConfigRequest;
|
||||
@@ -41,6 +42,7 @@ public class IntegrationConfigService {
|
||||
private final IntegrationConfigRepository repository;
|
||||
private final OwnershipService ownership;
|
||||
private final SecretMasker secretMasker;
|
||||
private final ResourceGrantRepository grantRepository;
|
||||
|
||||
// ---- commands ----
|
||||
|
||||
@@ -49,6 +51,13 @@ public class IntegrationConfigService {
|
||||
OwnerScope scope = request.scope() == null ? OwnerScope.USER : request.scope();
|
||||
IntegrationConfig cfg = new IntegrationConfig();
|
||||
cfg.setIntegrationType(require(request.integrationType(), "integrationType"));
|
||||
// S3 is infrastructure, not self-serve: no personal S3 for regular users. TEAM/SERVER
|
||||
// scopes are already restricted to admins/team owners by assignOwnership.
|
||||
if (cfg.getIntegrationType() == IntegrationType.S3
|
||||
&& scope == OwnerScope.USER
|
||||
&& !ownership.isAdmin(currentUser)) {
|
||||
throw forbidden("S3 connections can only be created by administrators or team owners");
|
||||
}
|
||||
cfg.setName(require(request.name(), "name"));
|
||||
cfg.setEnabled(request.enabled() == null || request.enabled());
|
||||
cfg.setLocked(request.locked() != null && request.locked());
|
||||
@@ -104,6 +113,8 @@ public class IntegrationConfigService {
|
||||
if (!ownership.canManage(TYPE, cfg, currentUser)) {
|
||||
throw forbidden("You cannot manage this integration");
|
||||
}
|
||||
// Drop grants sharing this config so they do not dangle as dead rows.
|
||||
grantRepository.deleteByResourceTypeAndResourceId(TYPE, String.valueOf(cfg.getId()));
|
||||
repository.delete(cfg);
|
||||
}
|
||||
|
||||
|
||||
+1
-2
@@ -1,4 +1,4 @@
|
||||
package stirling.software.saas.model;
|
||||
package stirling.software.proprietary.model;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -15,7 +15,6 @@ import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
/**
|
||||
+7
@@ -21,4 +21,11 @@ public class AiWorkflowResultFile {
|
||||
|
||||
@Schema(description = "MIME type of the file", example = "application/pdf")
|
||||
private String contentType;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Index into the request's fileInputs that this output was derived from, or null"
|
||||
+ " when it has no single source (e.g. a merge, or a generated file)."
|
||||
+ " Lets the client replace that input in place as a new version.")
|
||||
private Integer sourceIndex;
|
||||
}
|
||||
|
||||
+37
@@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestPart;
|
||||
@@ -47,6 +48,7 @@ import stirling.software.proprietary.policy.engine.PolicyRunHandle;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.PolicyValidator;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
@@ -86,6 +88,7 @@ public class PolicyController {
|
||||
private final PolicyManagementAuthority policyManagementAuthority;
|
||||
private final PolicyTriggerManager policyTriggerManager;
|
||||
private final PolicyOverviewService policyOverviewService;
|
||||
private final ProcessedLedger processedLedger;
|
||||
private final List<PolicyTrigger> policyTriggers;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final TempFileManager tempFileManager;
|
||||
@@ -213,6 +216,20 @@ public class PolicyController {
|
||||
return ResponseEntity.ok(saved);
|
||||
}
|
||||
|
||||
@PutMapping("/order")
|
||||
@Operation(
|
||||
summary = "Set the team's policy run order",
|
||||
description =
|
||||
"Persists the team-wide order policies run in, from the given ordered list of"
|
||||
+ " policy ids (position → order). The per-trigger order shown in the UI"
|
||||
+ " is this one sequence filtered by trigger. Team-leader/admin only;"
|
||||
+ " ids outside the caller's team are ignored.")
|
||||
public ResponseEntity<Void> reorderPolicies(@RequestBody List<String> orderedPolicyIds) {
|
||||
requirePolicyEditingAllowed();
|
||||
policyStore.reorder(policyAccessGuard.teamForNewPolicy(), orderedPolicyIds);
|
||||
return ResponseEntity.noContent().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Every {@code sourceId} a policy references must resolve to a source in the caller's team, so
|
||||
* a client can neither reference a non-existent source nor reach across teams to use another
|
||||
@@ -337,6 +354,7 @@ public class PolicyController {
|
||||
boolean accessible =
|
||||
policyStore.get(policyId).filter(policyAccessGuard::canAccess).isPresent();
|
||||
if (accessible && policyStore.delete(policyId)) {
|
||||
processedLedger.clearPolicy(policyId);
|
||||
// Cancel any now-orphaned folder watch promptly rather than leaving the WatchKey open
|
||||
// until the next reconcile sweep.
|
||||
policyTriggerManager.notifyPoliciesChanged();
|
||||
@@ -345,6 +363,25 @@ public class PolicyController {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
|
||||
@DeleteMapping("/{policyId}/processed-history")
|
||||
@Operation(
|
||||
summary = "Clear a policy's processed-file history",
|
||||
description =
|
||||
"Forgets which source files this policy has already processed, so its next"
|
||||
+ " sweep reprocesses everything currently in its sources. Does not"
|
||||
+ " touch the files themselves.")
|
||||
public ResponseEntity<Void> clearProcessedHistory(@PathVariable String policyId) {
|
||||
requirePolicyEditingAllowed();
|
||||
// Scope to the caller's team: a policy in another team reads as not-found.
|
||||
boolean accessible =
|
||||
policyStore.get(policyId).filter(policyAccessGuard::canAccess).isPresent();
|
||||
if (!accessible) {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
processedLedger.clearPolicy(policyId);
|
||||
return ResponseEntity.noContent().build();
|
||||
}
|
||||
|
||||
@PostMapping(value = "/{policyId}/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Run a stored policy",
|
||||
|
||||
+7
-1
@@ -35,6 +35,7 @@ import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
import stirling.software.proprietary.policy.model.WaitState;
|
||||
import stirling.software.proprietary.policy.output.OutputDelivery;
|
||||
import stirling.software.proprietary.policy.output.PolicyOutputSink;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
import stirling.software.proprietary.service.DownstreamEntitlementError;
|
||||
@@ -203,7 +204,12 @@ public class PolicyEngine {
|
||||
PolicyExecutionResult result =
|
||||
stepExecutor.execute(run.getDefinition(), inputs, listener);
|
||||
OutputSpec output = run.getDefinition().output();
|
||||
List<ResultFile> outputs = sinkFor(output).deliver(runId, result.files(), output);
|
||||
List<ResultFile> outputs =
|
||||
sinkFor(output)
|
||||
.deliver(
|
||||
new OutputDelivery(runId, run.getPolicyId()),
|
||||
result.files(),
|
||||
output);
|
||||
taskManager.setMultipleFileResults(runId, outputs);
|
||||
taskManager.setComplete(runId);
|
||||
run.complete(outputs);
|
||||
|
||||
+7
-3
@@ -8,7 +8,11 @@ import tools.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* Result of a {@link PolicyExecutor} run. {@code files} are final temp files (not yet stored).
|
||||
* {@code report}/{@code reportTool} carry the last step's structured report and its operation, or
|
||||
* null if no step produced one.
|
||||
* {@code origins} is parallel to {@code files}: each entry is the index into the original pipeline
|
||||
* inputs that the output traces back to, or {@code null} when it has no single source (e.g. a merge
|
||||
* combining several inputs, or a generated file). Callers use it to map an output back onto the
|
||||
* file it came from. {@code report}/{@code reportTool} carry the last step's structured report and
|
||||
* its operation, or null if no step produced one.
|
||||
*/
|
||||
public record PolicyExecutionResult(List<Resource> files, JsonNode report, String reportTool) {}
|
||||
public record PolicyExecutionResult(
|
||||
List<Resource> files, List<Integer> origins, JsonNode report, String reportTool) {}
|
||||
|
||||
+40
-9
@@ -58,6 +58,11 @@ public class PolicyExecutor {
|
||||
// payload the tool surfaced alongside or instead of a file.
|
||||
private record ToolResult(List<Resource> files, JsonNode report) {}
|
||||
|
||||
// A step's output files paired with each file's origin (the index into the original pipeline
|
||||
// inputs it traces back to, or null when it has no single source). Origins compose across steps
|
||||
// so the final result can be mapped back onto the files that entered the pipeline.
|
||||
private record StepOutput(List<Resource> files, List<Integer> origins, JsonNode report) {}
|
||||
|
||||
/**
|
||||
* Run every step in order, feeding each step's output into the next. Supporting files in {@code
|
||||
* inputs} bind to named file fields and never enter the document stream.
|
||||
@@ -75,6 +80,12 @@ public class PolicyExecutor {
|
||||
|
||||
List<Resource> currentFiles = inputs.primary();
|
||||
Map<String, List<Resource>> supportingFiles = inputs.supportingFiles();
|
||||
// Seed each input with its own index as origin; steps carry these through so the final
|
||||
// outputs can be traced back to the files that entered the pipeline.
|
||||
List<Integer> currentOrigins = new ArrayList<>();
|
||||
for (int k = 0; k < currentFiles.size(); k++) {
|
||||
currentOrigins.add(k);
|
||||
}
|
||||
// Last non-null report wins: the terminal step defines the output.
|
||||
JsonNode lastReport = null;
|
||||
String lastReportTool = null;
|
||||
@@ -87,8 +98,10 @@ public class PolicyExecutor {
|
||||
"Pipeline step " + (i + 1) + " has no operation");
|
||||
}
|
||||
listener.onStepStart(i + 1, steps.size(), operation);
|
||||
ToolResult stepResult = executeStep(step, currentFiles, supportingFiles);
|
||||
StepOutput stepResult =
|
||||
executeStep(step, currentFiles, currentOrigins, supportingFiles);
|
||||
currentFiles = stepResult.files();
|
||||
currentOrigins = stepResult.origins();
|
||||
if (stepResult.report() != null) {
|
||||
lastReport = stepResult.report();
|
||||
lastReportTool = operation;
|
||||
@@ -96,7 +109,7 @@ public class PolicyExecutor {
|
||||
listener.onStepComplete(i + 1, steps.size(), operation);
|
||||
}
|
||||
|
||||
return new PolicyExecutionResult(currentFiles, lastReport, lastReportTool);
|
||||
return new PolicyExecutionResult(currentFiles, currentOrigins, lastReport, lastReportTool);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -104,32 +117,50 @@ public class PolicyExecutor {
|
||||
* responses are unpacked so each inner file is its own result (e.g. split). For per-file
|
||||
* dispatch the first non-null report wins.
|
||||
*/
|
||||
private ToolResult executeStep(
|
||||
private StepOutput executeStep(
|
||||
PipelineStep step,
|
||||
List<Resource> inputFiles,
|
||||
List<Integer> inputOrigins,
|
||||
Map<String, List<Resource>> supportingFiles)
|
||||
throws IOException {
|
||||
requireAcceptedTypes(step.operation(), inputFiles);
|
||||
List<Resource> files = new ArrayList<>();
|
||||
List<Integer> origins = new ArrayList<>();
|
||||
JsonNode report = null;
|
||||
if (toolMetadataService.isMultiInput(step.operation())) {
|
||||
// One call over all inputs. The outputs derive from a single input only when exactly
|
||||
// one entered; otherwise (a genuine merge) there is no single source.
|
||||
ToolResult r = callEndpoint(step, inputFiles, supportingFiles);
|
||||
files.addAll(r.files());
|
||||
Integer origin = inputOrigins.size() == 1 ? inputOrigins.get(0) : null;
|
||||
for (Resource file : r.files()) {
|
||||
files.add(file);
|
||||
origins.add(origin);
|
||||
}
|
||||
report = r.report();
|
||||
} else if (inputFiles.isEmpty()) {
|
||||
ToolResult r = callEndpoint(step, List.of(), supportingFiles);
|
||||
files.addAll(r.files());
|
||||
for (Resource file : r.files()) {
|
||||
files.add(file);
|
||||
origins.add(null);
|
||||
}
|
||||
report = r.report();
|
||||
} else {
|
||||
for (Resource file : inputFiles) {
|
||||
ToolResult r = callEndpoint(step, List.of(file), supportingFiles);
|
||||
files.addAll(r.files());
|
||||
// One call per file: every output of this call inherits that input's origin, so a 1:1
|
||||
// op keeps its chain and a split (one input, many outputs) tags each output with the
|
||||
// same source.
|
||||
for (int k = 0; k < inputFiles.size(); k++) {
|
||||
Integer origin = inputOrigins.get(k);
|
||||
ToolResult r = callEndpoint(step, List.of(inputFiles.get(k)), supportingFiles);
|
||||
for (Resource file : r.files()) {
|
||||
files.add(file);
|
||||
origins.add(origin);
|
||||
}
|
||||
if (report == null) {
|
||||
report = r.report();
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ToolResult(files, report);
|
||||
return new StepOutput(files, origins, report);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+41
-8
@@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
@@ -27,7 +28,8 @@ import stirling.software.proprietary.policy.source.SourceStore;
|
||||
/**
|
||||
* Turns a policy's referenced sources into runs: each {@code sourceId} is resolved live to its
|
||||
* persisted {@link Source}, then to an {@link InputSpec}. Triggers decide <em>when</em> and call
|
||||
* {@link #run(Policy)}; the controller uses the supplied-input and ad-hoc entry points.
|
||||
* {@link #run(Policy)}; the controller uses the supplied-input and ad-hoc entry points. A {@link
|
||||
* SweepKind#FULL} sweep also reconciles the processed-file ledger against what is present.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -39,6 +41,12 @@ public class PolicyRunner {
|
||||
private final List<InputSource> inputSources;
|
||||
private final SourceStore sourceStore;
|
||||
private final SourceDocCounter docCounter;
|
||||
private final ProcessedLedger processedLedger;
|
||||
|
||||
/** Full-listing sweep: resolve every source, then reconcile the ledger. */
|
||||
public List<String> run(Policy policy) {
|
||||
return run(policy, SweepKind.FULL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
|
||||
@@ -47,15 +55,21 @@ public class PolicyRunner {
|
||||
* rest. Returns the ids of the runs it started (empty when sources yielded no work), so a
|
||||
* manual trigger can report back which runs to follow.
|
||||
*/
|
||||
public List<String> run(Policy policy) {
|
||||
public List<String> run(Policy policy, SweepKind sweep) {
|
||||
long sweepStart = System.currentTimeMillis();
|
||||
PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger);
|
||||
List<String> runIds = new ArrayList<>();
|
||||
List<String> sourceIds = policy.sourceIds();
|
||||
if (sourceIds.isEmpty()) {
|
||||
return List.of(startRun(policy, PolicyInputs.of(List.of()), unused -> {}));
|
||||
// Generator pipeline: one run with no input. Still fall through to the cleanup
|
||||
// below so rows recorded for its folder outputs are pruned like anything else,
|
||||
// instead of accumulating until the policy is deleted.
|
||||
runIds.add(startRun(policy, PolicyInputs.of(List.of()), unused -> {}));
|
||||
}
|
||||
List<String> runIds = new ArrayList<>();
|
||||
for (String sourceId : sourceIds) {
|
||||
Source source = sourceStore.get(sourceId).orElse(null);
|
||||
if (source == null) {
|
||||
// No veto: a deleted source's rows should age out via the cleanup below.
|
||||
log.warn("Policy {} references missing source {}; skipping", policy.id(), sourceId);
|
||||
continue;
|
||||
}
|
||||
@@ -65,9 +79,21 @@ public class PolicyRunner {
|
||||
sourceId,
|
||||
source.name(),
|
||||
policy.id());
|
||||
// Veto: a paused source's files cannot be stamped, so they must not be pruned.
|
||||
context.vetoCleanup();
|
||||
continue;
|
||||
}
|
||||
runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec()));
|
||||
runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec(), context));
|
||||
}
|
||||
if (context.cleanupAllowed()) {
|
||||
processedLedger.markSeen(policy.id(), context.presentIdentities());
|
||||
int removed = processedLedger.deleteUnseen(policy.id(), sweepStart);
|
||||
if (removed > 0) {
|
||||
log.debug(
|
||||
"Pruned {} ledger row(s) for files no longer present (policy {})",
|
||||
removed,
|
||||
policy.id());
|
||||
}
|
||||
}
|
||||
return runIds;
|
||||
}
|
||||
@@ -86,26 +112,33 @@ public class PolicyRunner {
|
||||
|
||||
/**
|
||||
* Resolves the source and starts a run per unit; records how many documents the source fed and
|
||||
* returns the ids of the runs started.
|
||||
* returns the ids of the runs started. Any source that could not be listed completely vetoes
|
||||
* this sweep's ledger cleanup.
|
||||
*/
|
||||
private List<String> pullAndRun(Policy policy, String sourceId, InputSpec spec) {
|
||||
private List<String> pullAndRun(
|
||||
Policy policy, String sourceId, InputSpec spec, PolicySweep context) {
|
||||
InputSource source = sourceFor(spec);
|
||||
if (source == null) {
|
||||
log.warn(
|
||||
"No input source for type '{}' (policy {}); skipping",
|
||||
spec.type(),
|
||||
policy.id());
|
||||
context.vetoCleanup();
|
||||
return List.of();
|
||||
}
|
||||
if (!source.listsExhaustively()) {
|
||||
context.vetoCleanup();
|
||||
}
|
||||
List<ResolvedInput> work;
|
||||
try {
|
||||
work = source.resolve(spec);
|
||||
work = source.resolve(spec, context);
|
||||
} catch (IOException | RuntimeException e) {
|
||||
log.warn(
|
||||
"Failed to resolve source '{}' for policy {}: {}",
|
||||
spec.type(),
|
||||
policy.id(),
|
||||
e.getMessage());
|
||||
context.vetoCleanup();
|
||||
return List.of();
|
||||
}
|
||||
List<String> runIds = new ArrayList<>();
|
||||
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import stirling.software.proprietary.policy.input.ResolveContext;
|
||||
import stirling.software.proprietary.policy.ledger.ClaimState;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedFileStatus;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
|
||||
/**
|
||||
* The {@link ResolveContext} for one policy sweep: scopes ledger calls to the policy, gathers the
|
||||
* present-identity union across sources, prefetches claim state in bulk so per-file claims skip
|
||||
* their row lookup, and vetoes presence cleanup when any source could not be listed completely
|
||||
* (pruning would wrongly forget its files).
|
||||
*/
|
||||
final class PolicySweep implements ResolveContext {
|
||||
|
||||
private final String policyId;
|
||||
private final SweepKind kind;
|
||||
private final ProcessedLedger ledger;
|
||||
private final Set<String> present = new HashSet<>();
|
||||
// Claim states loaded in bulk at reportPresent; a claim outside the prefetch falls back to a
|
||||
// single lookup. A stale entry cannot double-claim (the ledger re-checks every transition),
|
||||
// it can only defer a file to the next sweep.
|
||||
private final Map<String, ClaimState> prefetched = new HashMap<>();
|
||||
private final Set<String> prefetchedIdentities = new HashSet<>();
|
||||
private boolean cleanupVetoed;
|
||||
|
||||
PolicySweep(String policyId, SweepKind kind, ProcessedLedger ledger) {
|
||||
this.policyId = policyId;
|
||||
this.kind = kind;
|
||||
this.ledger = ledger;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized boolean claim(String identity, String gate, Supplier<String> contentHash) {
|
||||
ClaimState observed =
|
||||
prefetchedIdentities.contains(identity)
|
||||
? prefetched.get(identity)
|
||||
: ledger.statesFor(policyId, List.of(identity)).get(identity);
|
||||
boolean claimed = ledger.claim(policyId, identity, gate, contentHash, observed);
|
||||
if (claimed) {
|
||||
// A nested source surfacing the same file later in this sweep sees it in flight
|
||||
// without another lookup.
|
||||
prefetchedIdentities.add(identity);
|
||||
prefetched.put(identity, new ClaimState(ProcessedFileStatus.PROCESSING, gate, null));
|
||||
}
|
||||
return claimed;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(
|
||||
String identity, String finalGate, String finalContentHash, boolean success) {
|
||||
ledger.settle(policyId, identity, finalGate, finalContentHash, success);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allSettledDone(String identity) {
|
||||
// Deliberately not policy-scoped: consume deletion needs every claimant's consensus.
|
||||
return ledger.allSettledDone(identity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void reportPresent(Collection<String> identities) {
|
||||
if (kind == SweepKind.FULL) {
|
||||
present.addAll(identities);
|
||||
}
|
||||
prefetched.putAll(ledger.statesFor(policyId, identities));
|
||||
prefetchedIdentities.addAll(identities);
|
||||
}
|
||||
|
||||
synchronized void vetoCleanup() {
|
||||
cleanupVetoed = true;
|
||||
}
|
||||
|
||||
synchronized boolean cleanupAllowed() {
|
||||
return kind == SweepKind.FULL && !cleanupVetoed;
|
||||
}
|
||||
|
||||
synchronized Set<String> presentIdentities() {
|
||||
return Set.copyOf(present);
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
/**
|
||||
* How thorough a policy sweep is: {@link #FULL} (complete listing; also stamps presence and prunes
|
||||
* the ledger) or {@link #LIGHT} (event-driven; claims only, cost proportional to what changed).
|
||||
*/
|
||||
public enum SweepKind {
|
||||
FULL,
|
||||
LIGHT
|
||||
}
|
||||
+200
-74
@@ -1,12 +1,17 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
@@ -19,17 +24,20 @@ import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.ledger.FolderIdentities;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
|
||||
/**
|
||||
* Reads input files from a directory; each ready file is its own unit of work so one failure does
|
||||
* not affect the others.
|
||||
*
|
||||
* <p>Mode option: "consume" (default) claims each file by moving it into {@code
|
||||
* .stirling/processing} then routes it to {@code .stirling/done} or {@code .stirling/error}, so
|
||||
* each file runs once; "snapshot" reads without moving, so every run sees the full set. Readiness
|
||||
* is checked first so files mid-write are skipped.
|
||||
* Reads input files from a directory; each ready file is its own unit of work, claimed through the
|
||||
* {@link ResolveContext} ledger rather than moved aside, so nothing accumulates in a work
|
||||
* directory. Options: "mode" is "consume" (default: a processed file is removed once every policy
|
||||
* that claimed it has settled successfully and it is still the version that ran; failures stay in
|
||||
* place and are not retried until they change) or "snapshot" (stateless, every run sees the full
|
||||
* set); "recursive" descends into subdirectories; "identity" is "stat" (default, any size/mtime
|
||||
* change is a new version) or "hash" (content-verified, so a touch does not reprocess). Hidden
|
||||
* files and directories, including the legacy {@code .stirling} work dir, are never picked up, and
|
||||
* files mid-write are skipped by the readiness check.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -38,11 +46,6 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
public class FolderInputSource implements InputSource {
|
||||
|
||||
private static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
|
||||
// Bookkeeping lives under one hidden dir so the watched folder stays tidy.
|
||||
private static final String WORK_SUBDIR = ".stirling";
|
||||
private static final String PROCESSING_SUBDIR = "processing";
|
||||
private static final String DONE_SUBDIR = "done";
|
||||
private static final String ERROR_SUBDIR = "error";
|
||||
|
||||
private final FileReadinessChecker readinessChecker;
|
||||
private final FolderAccessGuard accessGuard;
|
||||
@@ -68,70 +71,195 @@ public class FolderInputSource implements InputSource {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec) throws IOException {
|
||||
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
|
||||
FolderConfig config = FolderConfig.from(spec.options());
|
||||
Path inputDir = accessGuard.requirePermitted(config.directory());
|
||||
if (!Files.isDirectory(inputDir)) {
|
||||
log.debug("Folder input dir does not exist: {}", inputDir);
|
||||
return List.of();
|
||||
// Fail rather than return empty: an unmounted drive must read as "could not list",
|
||||
// which vetoes the sweep's presence cleanup, not as "verifiably no files", which
|
||||
// would wipe the policy's history and reprocess everything on remount.
|
||||
throw new NoSuchFileException(
|
||||
inputDir.toString(), null, "input directory does not exist");
|
||||
}
|
||||
Path canonicalDir = FolderIdentities.canonicalDir(inputDir);
|
||||
List<Path> present = listFiles(inputDir, config.recursive());
|
||||
|
||||
if (config.snapshot()) {
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (Path file : present) {
|
||||
if (readinessChecker.isReady(file)) {
|
||||
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
|
||||
}
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
List<Path> ready = new ArrayList<>();
|
||||
try (Stream<Path> entries = Files.list(inputDir)) {
|
||||
entries.filter(Files::isRegularFile)
|
||||
.filter(readinessChecker::isReady)
|
||||
.forEach(ready::add);
|
||||
}
|
||||
ctx.reportPresent(
|
||||
present.stream()
|
||||
.map(file -> FolderIdentities.identity(canonicalDir, inputDir, file))
|
||||
.toList());
|
||||
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (Path file : ready) {
|
||||
if (config.snapshot()) {
|
||||
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
|
||||
} else {
|
||||
Path claimed = claim(inputDir, file);
|
||||
if (claimed == null) {
|
||||
continue; // another sweep/process grabbed it
|
||||
}
|
||||
work.add(
|
||||
new ResolvedInput(
|
||||
PolicyInputs.of(List.of(fileResource(claimed))),
|
||||
success -> route(inputDir, claimed, success)));
|
||||
for (Path file : present) {
|
||||
if (!readinessChecker.isReady(file)) {
|
||||
continue;
|
||||
}
|
||||
String identity = FolderIdentities.identity(canonicalDir, inputDir, file);
|
||||
MemoizedContentHash contentHash =
|
||||
config.hashIdentity() ? new MemoizedContentHash(file) : null;
|
||||
String gate;
|
||||
boolean claimed;
|
||||
try {
|
||||
gate = FolderIdentities.statGate(file);
|
||||
claimed = ctx.claim(identity, gate, contentHash);
|
||||
} catch (IOException | UncheckedIOException e) {
|
||||
log.debug("Could not read {} for its version: {}", file, e.getMessage());
|
||||
continue; // vanished or unreadable mid-sweep; the next sweep sees the truth
|
||||
}
|
||||
if (!claimed) {
|
||||
continue;
|
||||
}
|
||||
work.add(
|
||||
new ResolvedInput(
|
||||
PolicyInputs.of(List.of(fileResource(file))),
|
||||
success ->
|
||||
completeConsumed(
|
||||
ctx, identity, file, gate, contentHash, success)));
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
// Atomic move into processing/: only one sweep can win the claim, the rest see the file gone.
|
||||
private Path claim(Path inputDir, Path file) {
|
||||
/**
|
||||
* Settle at the version this run claimed - never a re-read, so a file replaced mid-run reads as
|
||||
* a new unclaimed version next sweep instead of being marked processed. Then remove the input
|
||||
* only when it is still the processed version (a mid-run replacement must survive) and every
|
||||
* policy that claimed it has settled DONE, so co-watching policies all read the original and
|
||||
* one failure parks the file for everyone. A failed run settles ERROR and never deletes; the
|
||||
* DONE row of a file that could not be deleted still stops reprocessing.
|
||||
*/
|
||||
private static void completeConsumed(
|
||||
ResolveContext ctx,
|
||||
String identity,
|
||||
Path file,
|
||||
String claimGate,
|
||||
MemoizedContentHash contentHash,
|
||||
boolean success) {
|
||||
ctx.settle(identity, claimGate, claimedHash(file, claimGate, contentHash), success);
|
||||
if (!success) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Path processingDir = workDir(inputDir, PROCESSING_SUBDIR);
|
||||
Files.createDirectories(processingDir);
|
||||
Path claimed = uniqueTarget(processingDir, file.getFileName().toString());
|
||||
Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE);
|
||||
return claimed;
|
||||
if (FolderIdentities.statGate(file).equals(claimGate) && ctx.allSettledDone(identity)) {
|
||||
Files.deleteIfExists(file);
|
||||
}
|
||||
} catch (NoSuchFileException alreadyGone) {
|
||||
// Removed by the user or a co-watching policy's own consensus delete: nothing to do.
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not claim {}: {}", file, e.getMessage());
|
||||
log.warn("Could not remove consumed input {}: {}", file, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The claimed version's content hash: the value computed during the claim when the ledger
|
||||
* consulted the verifier, else computed now while the file is still at the claimed gate (so the
|
||||
* hash describes what actually ran), else null. Always null in stat mode.
|
||||
*/
|
||||
private static String claimedHash(Path file, String claimGate, MemoizedContentHash hash) {
|
||||
if (hash == null) {
|
||||
return null;
|
||||
}
|
||||
String computed = hash.valueIfComputed();
|
||||
if (computed != null) {
|
||||
return computed;
|
||||
}
|
||||
try {
|
||||
if (FolderIdentities.statGate(file).equals(claimGate)) {
|
||||
return hash.get();
|
||||
}
|
||||
} catch (IOException | UncheckedIOException e) {
|
||||
log.debug("Could not hash {} at settle: {}", file, e.getMessage());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void route(Path inputDir, Path claimed, boolean success) {
|
||||
String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR;
|
||||
try {
|
||||
Path destDir = workDir(inputDir, subdir);
|
||||
Files.createDirectories(destDir);
|
||||
Files.move(
|
||||
claimed,
|
||||
uniqueTarget(destDir, claimed.getFileName().toString()),
|
||||
StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException e) {
|
||||
log.warn(
|
||||
"Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage());
|
||||
/** Lazy verification tier: invoked at most once by the ledger, retained for the settle. */
|
||||
private static final class MemoizedContentHash implements Supplier<String> {
|
||||
|
||||
private final Path file;
|
||||
private volatile String value;
|
||||
|
||||
private MemoizedContentHash(Path file) {
|
||||
this.file = file;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String get() {
|
||||
if (value == null) {
|
||||
try {
|
||||
value = FolderIdentities.contentHash(file);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
String valueIfComputed() {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private static Path workDir(Path inputDir, String subdir) {
|
||||
return inputDir.resolve(WORK_SUBDIR).resolve(subdir);
|
||||
/** Every non-hidden regular file in the source, readable or not. */
|
||||
private static List<Path> listFiles(Path inputDir, boolean recursive) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
if (!recursive) {
|
||||
try (Stream<Path> entries = Files.list(inputDir)) {
|
||||
entries.filter(Files::isRegularFile)
|
||||
.filter(file -> !hidden(file))
|
||||
.forEach(files::add);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
// Hidden subtrees are pruned wholesale; symlinked directories are not followed.
|
||||
Files.walkFileTree(
|
||||
inputDir,
|
||||
new SimpleFileVisitor<>() {
|
||||
@Override
|
||||
public FileVisitResult preVisitDirectory(
|
||||
Path dir, BasicFileAttributes attributes) {
|
||||
if (!dir.equals(inputDir) && hidden(dir)) {
|
||||
return FileVisitResult.SKIP_SUBTREE;
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) {
|
||||
if (attributes.isRegularFile() && !hidden(file)) {
|
||||
files.add(file);
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path file, IOException e) {
|
||||
log.debug("Skipping unreadable entry {}: {}", file, e.getMessage());
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
return files;
|
||||
}
|
||||
|
||||
private static boolean hidden(Path path) {
|
||||
Path name = path.getFileName();
|
||||
if (name != null && name.toString().startsWith(".")) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
return Files.isHidden(path);
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static Resource fileResource(Path path) {
|
||||
@@ -144,27 +272,15 @@ public class FolderInputSource implements InputSource {
|
||||
};
|
||||
}
|
||||
|
||||
private static Path uniqueTarget(Path dir, String filename) {
|
||||
Path candidate = dir.resolve(filename);
|
||||
if (!Files.exists(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
int dot = filename.lastIndexOf('.');
|
||||
String base = dot < 0 ? filename : filename.substring(0, dot);
|
||||
String ext = dot < 0 ? "" : filename.substring(dot);
|
||||
for (int n = 1; ; n++) {
|
||||
Path next = dir.resolve(base + " (" + n + ")" + ext);
|
||||
if (!Files.exists(next)) {
|
||||
return next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
record FolderConfig(Path directory, boolean snapshot) {
|
||||
record FolderConfig(Path directory, boolean snapshot, boolean recursive, boolean hashIdentity) {
|
||||
|
||||
private static final String DIRECTORY_OPTION = "directory";
|
||||
private static final String MODE_OPTION = "mode";
|
||||
private static final String MODE_SNAPSHOT = "snapshot";
|
||||
private static final String RECURSIVE_OPTION = "recursive";
|
||||
private static final String IDENTITY_OPTION = "identity";
|
||||
private static final String IDENTITY_STAT = "stat";
|
||||
private static final String IDENTITY_HASH = "hash";
|
||||
|
||||
static FolderConfig from(Map<String, Object> options) {
|
||||
Object directory = options.get(DIRECTORY_OPTION);
|
||||
@@ -173,7 +289,17 @@ public class FolderInputSource implements InputSource {
|
||||
}
|
||||
Object mode = options.get(MODE_OPTION);
|
||||
boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString());
|
||||
return new FolderConfig(Path.of(directory.toString()), snapshot);
|
||||
Object recursive = options.get(RECURSIVE_OPTION);
|
||||
boolean recurse = recursive != null && Boolean.parseBoolean(recursive.toString());
|
||||
Object identity = options.get(IDENTITY_OPTION);
|
||||
boolean hash = identity != null && IDENTITY_HASH.equals(identity.toString());
|
||||
if (identity != null
|
||||
&& !IDENTITY_STAT.equals(identity.toString())
|
||||
&& !IDENTITY_HASH.equals(identity.toString())) {
|
||||
throw new IllegalArgumentException(
|
||||
"folder input 'identity' must be 'stat' or 'hash'");
|
||||
}
|
||||
return new FolderConfig(Path.of(directory.toString()), snapshot, recurse, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+14
-2
@@ -24,9 +24,21 @@ public interface InputSource {
|
||||
|
||||
/**
|
||||
* Resolve the spec into zero or more units of work, each carrying one run's files and a
|
||||
* completion hook. Empty list means nothing to run right now.
|
||||
* completion hook. Empty list means nothing to run right now. Discovery is read-only - files
|
||||
* stay where the user put them; "already processed" is tracked through {@code ctx} (claim on
|
||||
* pickup, settle on completion, report what is present so stale ledger rows can be pruned).
|
||||
*/
|
||||
List<ResolvedInput> resolve(InputSpec spec) throws IOException;
|
||||
List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException;
|
||||
|
||||
/**
|
||||
* Whether {@link #resolve} observes everything in the source (a complete listing) rather than
|
||||
* e.g. only what events surfaced. Presence cleanup of the ledger is skipped for the whole
|
||||
* policy unless every enabled source says true - wrongly pruning history would reprocess a
|
||||
* whole folder, while keeping a few stale rows costs nothing.
|
||||
*/
|
||||
default boolean listsExhaustively() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filesystem dirs this source draws from, for the folder-watch trigger. Advisory: resolving is
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* A source's policy-scoped window onto the processed-file ledger for one sweep. Thread-safe and
|
||||
* valid for the lifetime of the work units the source issued ({@link #settle} fires from async run
|
||||
* completions).
|
||||
*/
|
||||
public interface ResolveContext {
|
||||
|
||||
/**
|
||||
* Atomically claim a file at its current version; true means this sweep runs it. A null {@code
|
||||
* contentHash} makes any gate change a new version; a non-null supplier is invoked at most
|
||||
* once, only on a gate mismatch, and a matching hash refreshes the stored gate instead of
|
||||
* reprocessing. Supplier exceptions propagate.
|
||||
*/
|
||||
boolean claim(String identity, String gate, Supplier<String> contentHash);
|
||||
|
||||
/** Record a claimed file's outcome at its final version ({@code finalContentHash} nullable). */
|
||||
void settle(String identity, String finalGate, String finalContentHash, boolean success);
|
||||
|
||||
/**
|
||||
* Whether every policy holding a ledger row for this identity has settled it DONE. Cross-policy
|
||||
* by design: consume-mode deletion is a consensus of all claimants, so a shared input is
|
||||
* removed only once nobody still needs it (in-flight, failed, and interrupted rows all veto).
|
||||
*/
|
||||
boolean allSettledDone(String identity);
|
||||
|
||||
/**
|
||||
* Report every identity present right now, readable or not; feeds presence cleanup of rows
|
||||
* whose file is gone.
|
||||
*/
|
||||
void reportPresent(Collection<String> identities);
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
/**
|
||||
* A row's claim-relevant state as read by {@link ProcessedLedger#statesFor}: what a sweep observed
|
||||
* before deciding a claim. May be stale by the time the claim runs; every ledger transition
|
||||
* re-checks the observed state in its WHERE clause, so staleness defers a claim to a later sweep
|
||||
* rather than double-running one.
|
||||
*/
|
||||
public record ClaimState(ProcessedFileStatus status, String gate, String contentHash) {}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
|
||||
import stirling.software.proprietary.billing.ContentHasher;
|
||||
|
||||
/**
|
||||
* The folder backend's identity and version scheme, shared by {@code FolderInputSource} and {@code
|
||||
* FolderOutputSink} so outputs are recorded under exactly the identity the next scan derives.
|
||||
* Directories are canonicalised with {@code toRealPath()} so symlinked aliases agree.
|
||||
*/
|
||||
public final class FolderIdentities {
|
||||
|
||||
private FolderIdentities() {}
|
||||
|
||||
/** Canonical form of a configured directory; resolves symlinks, so the dir must exist. */
|
||||
public static Path canonicalDir(Path dir) throws IOException {
|
||||
return dir.toRealPath();
|
||||
}
|
||||
|
||||
/** Identity of {@code file} under {@code dir}: its path re-rooted onto the canonical dir. */
|
||||
public static String identity(Path canonicalDir, Path dir, Path file) {
|
||||
return canonicalDir.resolve(dir.relativize(file)).normalize().toString();
|
||||
}
|
||||
|
||||
/** The cheap version gate: a change to content length or mtime means "look closer". */
|
||||
public static String statGate(Path file) throws IOException {
|
||||
BasicFileAttributes attributes = Files.readAttributes(file, BasicFileAttributes.class);
|
||||
return attributes.size() + ":" + attributes.lastModifiedTime().toMillis();
|
||||
}
|
||||
|
||||
/**
|
||||
* The strong version token: distinguishes a real change from a touch, at the cost of a read.
|
||||
*/
|
||||
public static String contentHash(Path file) throws IOException {
|
||||
return ContentHasher.sha256(file);
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import stirling.software.proprietary.billing.ContentHasher;
|
||||
|
||||
/**
|
||||
* Fixed-width key form of a source-owned identity, so any identity length fits the ledger's primary
|
||||
* key. Backend-agnostic: every source type's identities are keyed through here, which is why this
|
||||
* does not live with the folder backend's {@link FolderIdentities}.
|
||||
*/
|
||||
public final class IdentityHasher {
|
||||
|
||||
private IdentityHasher() {}
|
||||
|
||||
public static String identityHash(String identity) {
|
||||
return ContentHasher.sha256(identity.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
}
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* In-memory {@link ProcessedLedger} for tests and DB-less wiring; kept semantically identical to
|
||||
* {@code JpaProcessedLedger} by the shared contract test.
|
||||
*/
|
||||
public class InProcessProcessedLedger implements ProcessedLedger {
|
||||
|
||||
private final Map<String, Map<String, Row>> rowsByPolicy = new HashMap<>();
|
||||
private final Supplier<Long> nowMillis;
|
||||
|
||||
public InProcessProcessedLedger() {
|
||||
this(System::currentTimeMillis);
|
||||
}
|
||||
|
||||
public InProcessProcessedLedger(Supplier<Long> nowMillis) {
|
||||
this.nowMillis = nowMillis;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Map<String, ClaimState> statesFor(
|
||||
String policyId, Collection<String> identities) {
|
||||
Map<String, Row> rows = rowsByPolicy.getOrDefault(policyId, Map.of());
|
||||
Map<String, ClaimState> states = new HashMap<>();
|
||||
for (String identity : identities) {
|
||||
Row row = rows.get(identity);
|
||||
if (row != null) {
|
||||
states.put(identity, new ClaimState(row.status, row.gate, row.contentHash));
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
// Single-lock store: the live row is never staler than any observed snapshot, so decide
|
||||
// against it directly; the conditional updates of the JPA ledger yield the same outcomes.
|
||||
@Override
|
||||
public synchronized boolean claim(
|
||||
String policyId,
|
||||
String identity,
|
||||
String gate,
|
||||
Supplier<String> contentHash,
|
||||
ClaimState observed) {
|
||||
Map<String, Row> rows = rowsByPolicy.computeIfAbsent(policyId, key -> new HashMap<>());
|
||||
long now = nowMillis.get();
|
||||
Row row = rows.get(identity);
|
||||
if (row == null) {
|
||||
String hash = contentHash == null ? null : contentHash.get();
|
||||
rows.put(identity, new Row(gate, hash, ProcessedFileStatus.PROCESSING, 1, now));
|
||||
return true;
|
||||
}
|
||||
if (row.status == ProcessedFileStatus.PROCESSING) {
|
||||
return false;
|
||||
}
|
||||
if (gate.equals(row.gate)) {
|
||||
if (row.status == ProcessedFileStatus.INTERRUPTED && row.attempts < MAX_ATTEMPTS) {
|
||||
row.status = ProcessedFileStatus.PROCESSING;
|
||||
row.attempts++;
|
||||
row.lastSeen = now;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (contentHash == null) {
|
||||
row.gate = gate;
|
||||
row.contentHash = null;
|
||||
row.status = ProcessedFileStatus.PROCESSING;
|
||||
row.attempts = 1;
|
||||
row.lastSeen = now;
|
||||
return true;
|
||||
}
|
||||
String hash = contentHash.get();
|
||||
if (Objects.equals(hash, row.contentHash)) {
|
||||
if (row.status == ProcessedFileStatus.INTERRUPTED && row.attempts < MAX_ATTEMPTS) {
|
||||
row.gate = gate;
|
||||
row.status = ProcessedFileStatus.PROCESSING;
|
||||
row.attempts++;
|
||||
row.lastSeen = now;
|
||||
return true;
|
||||
}
|
||||
if (row.status != ProcessedFileStatus.INTERRUPTED) {
|
||||
row.gate = gate;
|
||||
row.lastSeen = now;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
row.gate = gate;
|
||||
row.contentHash = hash;
|
||||
row.status = ProcessedFileStatus.PROCESSING;
|
||||
row.attempts = 1;
|
||||
row.lastSeen = now;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void settle(
|
||||
String policyId,
|
||||
String identity,
|
||||
String finalGate,
|
||||
String finalContentHash,
|
||||
boolean success) {
|
||||
upsertSettled(
|
||||
policyId,
|
||||
identity,
|
||||
finalGate,
|
||||
finalContentHash,
|
||||
success ? ProcessedFileStatus.DONE : ProcessedFileStatus.ERROR);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void recordOutput(
|
||||
String policyId, String identity, String gate, String contentHash) {
|
||||
upsertSettled(policyId, identity, gate, contentHash, ProcessedFileStatus.DONE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void forgetOutput(String policyId, String identity, String gate) {
|
||||
Map<String, Row> rows = rowsByPolicy.get(policyId);
|
||||
if (rows == null) {
|
||||
return;
|
||||
}
|
||||
Row row = rows.get(identity);
|
||||
if (row != null && row.status == ProcessedFileStatus.DONE && gate.equals(row.gate)) {
|
||||
rows.remove(identity);
|
||||
}
|
||||
}
|
||||
|
||||
private void upsertSettled(
|
||||
String policyId,
|
||||
String identity,
|
||||
String gate,
|
||||
String contentHash,
|
||||
ProcessedFileStatus status) {
|
||||
Map<String, Row> rows = rowsByPolicy.computeIfAbsent(policyId, key -> new HashMap<>());
|
||||
long now = nowMillis.get();
|
||||
Row row = rows.get(identity);
|
||||
if (row == null) {
|
||||
rows.put(identity, new Row(gate, contentHash, status, 1, now));
|
||||
return;
|
||||
}
|
||||
row.gate = gate;
|
||||
row.contentHash = contentHash;
|
||||
row.status = status;
|
||||
row.lastSeen = now;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized boolean allSettledDone(String identity) {
|
||||
for (Map<String, Row> rows : rowsByPolicy.values()) {
|
||||
Row row = rows.get(identity);
|
||||
if (row != null && row.status != ProcessedFileStatus.DONE) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void markSeen(String policyId, Collection<String> identities) {
|
||||
Map<String, Row> rows = rowsByPolicy.get(policyId);
|
||||
if (rows == null) {
|
||||
return;
|
||||
}
|
||||
long now = nowMillis.get();
|
||||
for (String identity : identities) {
|
||||
Row row = rows.get(identity);
|
||||
if (row != null) {
|
||||
row.lastSeen = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized int deleteUnseen(String policyId, long seenSinceMillis) {
|
||||
Map<String, Row> rows = rowsByPolicy.get(policyId);
|
||||
if (rows == null) {
|
||||
return 0;
|
||||
}
|
||||
int before = rows.size();
|
||||
rows.values()
|
||||
.removeIf(
|
||||
row ->
|
||||
row.lastSeen < seenSinceMillis
|
||||
&& row.status != ProcessedFileStatus.PROCESSING);
|
||||
return before - rows.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void clearPolicy(String policyId) {
|
||||
rowsByPolicy.remove(policyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void recoverInterrupted() {
|
||||
for (Map<String, Row> rows : rowsByPolicy.values()) {
|
||||
for (Row row : rows.values()) {
|
||||
if (row.status == ProcessedFileStatus.PROCESSING) {
|
||||
row.status = ProcessedFileStatus.INTERRUPTED;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class Row {
|
||||
private String gate;
|
||||
private String contentHash;
|
||||
private ProcessedFileStatus status;
|
||||
private int attempts;
|
||||
private long lastSeen;
|
||||
|
||||
private Row(
|
||||
String gate,
|
||||
String contentHash,
|
||||
ProcessedFileStatus status,
|
||||
int attempts,
|
||||
long lastSeen) {
|
||||
this.gate = gate;
|
||||
this.contentHash = contentHash;
|
||||
this.status = status;
|
||||
this.attempts = attempts;
|
||||
this.lastSeen = lastSeen;
|
||||
}
|
||||
}
|
||||
}
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.dao.DataIntegrityViolationException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Durable {@link ProcessedLedger}; the runtime bean. A fresh claim is a flushed insert so a
|
||||
* concurrent winner surfaces as a constraint violation; every other transition is a conditional
|
||||
* update that re-checks the observed state, so a lost race reports 0 rows and the caller skips.
|
||||
* Boot recovery assumes the single node the folder-watch trigger assumes: runs live in memory, so
|
||||
* after a restart every PROCESSING row is stale.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class JpaProcessedLedger implements ProcessedLedger {
|
||||
|
||||
private static final int STAMP_CHUNK = 500;
|
||||
|
||||
private final ProcessedFileRepository repository;
|
||||
private final Supplier<Long> nowMillis;
|
||||
|
||||
@Autowired
|
||||
public JpaProcessedLedger(ProcessedFileRepository repository) {
|
||||
this(repository, System::currentTimeMillis);
|
||||
}
|
||||
|
||||
// Clock seam so tests can pin "now"; the runtime bean uses the wall clock above.
|
||||
JpaProcessedLedger(ProcessedFileRepository repository, Supplier<Long> nowMillis) {
|
||||
this.repository = repository;
|
||||
this.nowMillis = nowMillis;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ClaimState> statesFor(String policyId, Collection<String> identities) {
|
||||
if (identities.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, String> identityByHash = new HashMap<>();
|
||||
for (String identity : identities) {
|
||||
identityByHash.put(IdentityHasher.identityHash(identity), identity);
|
||||
}
|
||||
Map<String, ClaimState> states = new HashMap<>();
|
||||
List<String> hashes = List.copyOf(identityByHash.keySet());
|
||||
for (int from = 0; from < hashes.size(); from += STAMP_CHUNK) {
|
||||
List<ProcessedFileEntity> rows =
|
||||
repository.findByPolicyIdAndIdentityHashIn(
|
||||
policyId,
|
||||
hashes.subList(from, Math.min(from + STAMP_CHUNK, hashes.size())));
|
||||
for (ProcessedFileEntity row : rows) {
|
||||
states.put(
|
||||
identityByHash.get(row.getIdentityHash()),
|
||||
new ClaimState(row.getStatus(), row.getSignature(), row.getContentHash()));
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean claim(
|
||||
String policyId,
|
||||
String identity,
|
||||
String gate,
|
||||
Supplier<String> contentHash,
|
||||
ClaimState observed) {
|
||||
String identityHash = IdentityHasher.identityHash(identity);
|
||||
long now = nowMillis.get();
|
||||
if (observed == null) {
|
||||
try {
|
||||
repository.saveAndFlush(
|
||||
new ProcessedFileEntity(
|
||||
policyId,
|
||||
identityHash,
|
||||
identity,
|
||||
gate,
|
||||
contentHash == null ? null : contentHash.get(),
|
||||
ProcessedFileStatus.PROCESSING,
|
||||
now));
|
||||
return true;
|
||||
} catch (DataIntegrityViolationException concurrentClaim) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (observed.status() == ProcessedFileStatus.PROCESSING) {
|
||||
return false;
|
||||
}
|
||||
if (gate.equals(observed.gate())) {
|
||||
if (observed.status() == ProcessedFileStatus.INTERRUPTED) {
|
||||
return repository.retryInterruptedAtGate(
|
||||
policyId, identityHash, gate, MAX_ATTEMPTS, now)
|
||||
> 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (contentHash == null) {
|
||||
return repository.reclaimAtNewGate(policyId, identityHash, gate, now) > 0;
|
||||
}
|
||||
String hash = contentHash.get();
|
||||
if (hash.equals(observed.contentHash())) {
|
||||
if (observed.status() == ProcessedFileStatus.INTERRUPTED) {
|
||||
return repository.retryInterruptedSameContent(
|
||||
policyId, identityHash, gate, hash, MAX_ATTEMPTS, now)
|
||||
> 0;
|
||||
}
|
||||
repository.refreshGate(policyId, identityHash, gate, hash, now);
|
||||
return false;
|
||||
}
|
||||
return repository.reclaimAtNewContent(policyId, identityHash, gate, hash, now) > 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(
|
||||
String policyId,
|
||||
String identity,
|
||||
String finalGate,
|
||||
String finalContentHash,
|
||||
boolean success) {
|
||||
upsertSettled(
|
||||
policyId,
|
||||
identity,
|
||||
finalGate,
|
||||
finalContentHash,
|
||||
success ? ProcessedFileStatus.DONE : ProcessedFileStatus.ERROR);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordOutput(String policyId, String identity, String gate, String contentHash) {
|
||||
upsertSettled(policyId, identity, gate, contentHash, ProcessedFileStatus.DONE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void forgetOutput(String policyId, String identity, String gate) {
|
||||
repository.deleteDoneAt(policyId, IdentityHasher.identityHash(identity), gate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Settle-or-insert: the row may have been presence-cleaned mid-run, and an output row may be
|
||||
* brand new.
|
||||
*/
|
||||
private void upsertSettled(
|
||||
String policyId,
|
||||
String identity,
|
||||
String gate,
|
||||
String contentHash,
|
||||
ProcessedFileStatus status) {
|
||||
String identityHash = IdentityHasher.identityHash(identity);
|
||||
long now = nowMillis.get();
|
||||
if (repository.settle(policyId, identityHash, gate, contentHash, status, now) > 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ProcessedFileEntity row =
|
||||
new ProcessedFileEntity(
|
||||
policyId, identityHash, identity, gate, contentHash, status, now);
|
||||
repository.saveAndFlush(row);
|
||||
} catch (DataIntegrityViolationException concurrentInsert) {
|
||||
repository.settle(policyId, identityHash, gate, contentHash, status, now);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allSettledDone(String identity) {
|
||||
return !repository.existsByIdentityHashAndStatusNot(
|
||||
IdentityHasher.identityHash(identity), ProcessedFileStatus.DONE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markSeen(String policyId, Collection<String> identities) {
|
||||
if (identities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<String> hashes = identities.stream().map(IdentityHasher::identityHash).toList();
|
||||
long now = nowMillis.get();
|
||||
for (int from = 0; from < hashes.size(); from += STAMP_CHUNK) {
|
||||
repository.stampSeen(
|
||||
policyId,
|
||||
hashes.subList(from, Math.min(from + STAMP_CHUNK, hashes.size())),
|
||||
now);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteUnseen(String policyId, long seenSinceMillis) {
|
||||
return repository.deleteUnseen(policyId, seenSinceMillis);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clearPolicy(String policyId) {
|
||||
repository.deleteByPolicy(policyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void recoverInterrupted() {
|
||||
int recovered = repository.markAllProcessingInterrupted(nowMillis.get());
|
||||
if (recovered > 0) {
|
||||
log.info("Recovered {} policy input file(s) interrupted by shutdown", recovered);
|
||||
}
|
||||
}
|
||||
}
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import org.springframework.data.domain.Persistable;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.IdClass;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.Table;
|
||||
import jakarta.persistence.Transient;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* One processed-file ledger row: the version a policy last settled a file at, and where it is in
|
||||
* the claim lifecycle. Keyed by SHA-256 of the source-owned identity so any identity length fits a
|
||||
* fixed-width index. {@code isNew} is always true: the entity is only saved for fresh inserts
|
||||
* (everything else is a conditional update), so a lost insert race surfaces as a constraint
|
||||
* violation rather than a silent merge.
|
||||
*/
|
||||
@Entity
|
||||
@Table(
|
||||
name = "policy_processed_files",
|
||||
indexes = {
|
||||
// presence cleanup: delete this policy's rows unseen since the sweep began
|
||||
@Index(name = "idx_processed_files_policy_seen", columnList = "policy_id, last_seen"),
|
||||
// cross-policy deletion consensus: existsByIdentityHashAndStatusNot filters
|
||||
// identity_hash on its own, so it cannot ride the (policy_id, identity_hash) PK
|
||||
@Index(name = "idx_processed_files_identity", columnList = "identity_hash")
|
||||
})
|
||||
@IdClass(ProcessedFileId.class)
|
||||
@NoArgsConstructor
|
||||
@Getter
|
||||
@Setter
|
||||
public class ProcessedFileEntity implements Serializable, Persistable<ProcessedFileId> {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@Column(name = "policy_id")
|
||||
private String policyId;
|
||||
|
||||
@Id
|
||||
@Column(name = "identity_hash", length = 64)
|
||||
private String identityHash;
|
||||
|
||||
@Column(name = "identity", length = 4096)
|
||||
private String identity;
|
||||
|
||||
@Column(name = "signature")
|
||||
private String signature;
|
||||
|
||||
@Column(name = "content_hash", length = 64)
|
||||
private String contentHash;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "status", length = 16)
|
||||
private ProcessedFileStatus status;
|
||||
|
||||
@Column(name = "attempts")
|
||||
private int attempts;
|
||||
|
||||
@Column(name = "last_seen")
|
||||
private long lastSeen;
|
||||
|
||||
@Column(name = "updated_at")
|
||||
private long updatedAt;
|
||||
|
||||
public ProcessedFileEntity(
|
||||
String policyId,
|
||||
String identityHash,
|
||||
String identity,
|
||||
String signature,
|
||||
String contentHash,
|
||||
ProcessedFileStatus status,
|
||||
long nowMillis) {
|
||||
this.policyId = policyId;
|
||||
this.identityHash = identityHash;
|
||||
this.identity = identity;
|
||||
this.signature = signature;
|
||||
this.contentHash = contentHash;
|
||||
this.status = status;
|
||||
this.attempts = 1;
|
||||
this.lastSeen = nowMillis;
|
||||
this.updatedAt = nowMillis;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transient
|
||||
public ProcessedFileId getId() {
|
||||
return new ProcessedFileId(policyId, identityHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transient
|
||||
public boolean isNew() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
/** Composite key for {@link ProcessedFileEntity}: one row per policy per file identity. */
|
||||
public class ProcessedFileId implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private String policyId;
|
||||
private String identityHash;
|
||||
|
||||
public ProcessedFileId() {}
|
||||
|
||||
public ProcessedFileId(String policyId, String identityHash) {
|
||||
this.policyId = policyId;
|
||||
this.identityHash = identityHash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (!(o instanceof ProcessedFileId other)) {
|
||||
return false;
|
||||
}
|
||||
return Objects.equals(policyId, other.policyId)
|
||||
&& Objects.equals(identityHash, other.identityHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(policyId, identityHash);
|
||||
}
|
||||
}
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* Conditional updates for the processed-file ledger: each claim variant re-checks in its WHERE
|
||||
* clause the state it was decided against, so a racing claim loses cleanly with 0 rows updated.
|
||||
* Transactional per call so the ledger can run them without an enclosing transaction.
|
||||
*/
|
||||
@Repository
|
||||
public interface ProcessedFileRepository
|
||||
extends JpaRepository<ProcessedFileEntity, ProcessedFileId> {
|
||||
|
||||
/**
|
||||
* Re-claim a settled row at a new gate without content verification; clears the stored hash,
|
||||
* which described content this claim never checked.
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING,"
|
||||
+ " e.signature = :gate, e.contentHash = null, e.attempts = 1,"
|
||||
+ " e.lastSeen = :now, e.updatedAt = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash = :identityHash"
|
||||
+ " and e.status <>"
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING"
|
||||
+ " and e.signature <> :gate")
|
||||
int reclaimAtNewGate(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate,
|
||||
@Param("now") long now);
|
||||
|
||||
/** Re-claim a settled row whose content verifiably changed (or was never hashed). */
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING,"
|
||||
+ " e.signature = :gate, e.contentHash = :contentHash, e.attempts = 1,"
|
||||
+ " e.lastSeen = :now, e.updatedAt = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash = :identityHash"
|
||||
+ " and e.status <>"
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING"
|
||||
+ " and (e.contentHash is null or e.contentHash <> :contentHash)")
|
||||
int reclaimAtNewContent(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate,
|
||||
@Param("contentHash") String contentHash,
|
||||
@Param("now") long now);
|
||||
|
||||
/** The gate moved but the content did not: track the new gate without changing status. */
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.signature = :gate, e.lastSeen = :now,"
|
||||
+ " e.updatedAt = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash = :identityHash"
|
||||
+ " and e.status <>"
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING"
|
||||
+ " and e.contentHash = :contentHash and e.signature <> :gate")
|
||||
int refreshGate(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate,
|
||||
@Param("contentHash") String contentHash,
|
||||
@Param("now") long now);
|
||||
|
||||
/** Bounded retry of an INTERRUPTED row at the same gate. */
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING,"
|
||||
+ " e.attempts = e.attempts + 1, e.lastSeen = :now, e.updatedAt = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash = :identityHash"
|
||||
+ " and e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.INTERRUPTED"
|
||||
+ " and e.signature = :gate and e.attempts < :maxAttempts")
|
||||
int retryInterruptedAtGate(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate,
|
||||
@Param("maxAttempts") int maxAttempts,
|
||||
@Param("now") long now);
|
||||
|
||||
/** Bounded retry of an INTERRUPTED row whose gate moved but whose content is unchanged. */
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING,"
|
||||
+ " e.signature = :gate, e.attempts = e.attempts + 1, e.lastSeen = :now,"
|
||||
+ " e.updatedAt = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash = :identityHash"
|
||||
+ " and e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.INTERRUPTED"
|
||||
+ " and e.contentHash = :contentHash and e.attempts < :maxAttempts")
|
||||
int retryInterruptedSameContent(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate,
|
||||
@Param("contentHash") String contentHash,
|
||||
@Param("maxAttempts") int maxAttempts,
|
||||
@Param("now") long now);
|
||||
|
||||
/**
|
||||
* Unconditional settle (only the claiming run settles a row); returns 0 when the row was
|
||||
* removed mid-run so the caller re-inserts.
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.status = :status, e.signature = :gate,"
|
||||
+ " e.contentHash = :contentHash, e.lastSeen = :now, e.updatedAt = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash = :identityHash")
|
||||
int settle(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate,
|
||||
@Param("contentHash") String contentHash,
|
||||
@Param("status") ProcessedFileStatus status,
|
||||
@Param("now") long now);
|
||||
|
||||
/** Whether any policy's row at this identity is in a state other than {@code status}. */
|
||||
boolean existsByIdentityHashAndStatusNot(String identityHash, ProcessedFileStatus status);
|
||||
|
||||
/** One policy's rows across a chunk of identity hashes, for a sweep's claim snapshot. */
|
||||
List<ProcessedFileEntity> findByPolicyIdAndIdentityHashIn(
|
||||
String policyId, Collection<String> identityHashes);
|
||||
|
||||
/**
|
||||
* Remove an output record whose rename never landed, only while still settled exactly as
|
||||
* recorded; a row a claim has since taken over is left alone.
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"delete from ProcessedFileEntity e where e.policyId = :policyId"
|
||||
+ " and e.identityHash = :identityHash and e.signature = :gate"
|
||||
+ " and e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.DONE")
|
||||
int deleteDoneAt(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHash") String identityHash,
|
||||
@Param("gate") String gate);
|
||||
|
||||
/** Stamp presence for the given identities; chunked by the caller for very large folders. */
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.lastSeen = :now"
|
||||
+ " where e.policyId = :policyId and e.identityHash in :identityHashes")
|
||||
int stampSeen(
|
||||
@Param("policyId") String policyId,
|
||||
@Param("identityHashes") Collection<String> identityHashes,
|
||||
@Param("now") long now);
|
||||
|
||||
/**
|
||||
* Presence cleanup: remove rows not stamped since the sweep began, keeping in-flight claims.
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"delete from ProcessedFileEntity e where e.policyId = :policyId"
|
||||
+ " and e.lastSeen < :cutoff and e.status <>"
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING")
|
||||
int deleteUnseen(@Param("policyId") String policyId, @Param("cutoff") long cutoff);
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query("delete from ProcessedFileEntity e where e.policyId = :policyId")
|
||||
int deleteByPolicy(@Param("policyId") String policyId);
|
||||
|
||||
/** Boot recovery: after a restart every PROCESSING row is stale (single node). */
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(
|
||||
"update ProcessedFileEntity e set e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.INTERRUPTED,"
|
||||
+ " e.updatedAt = :now"
|
||||
+ " where e.status ="
|
||||
+ " stirling.software.proprietary.policy.ledger.ProcessedFileStatus.PROCESSING")
|
||||
int markAllProcessingInterrupted(@Param("now") long now);
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
/** Lifecycle of one {@code (policy, file)} ledger row. */
|
||||
public enum ProcessedFileStatus {
|
||||
|
||||
/** Claimed; a run is in flight. */
|
||||
PROCESSING,
|
||||
|
||||
/** Run completed at this version. */
|
||||
DONE,
|
||||
|
||||
/** Run failed; skipped until the file changes (clear-history is the manual retry). */
|
||||
ERROR,
|
||||
|
||||
/** Was PROCESSING when the JVM died; retried a bounded number of times. */
|
||||
INTERRUPTED
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Remembers which files a policy has processed, one row per {@code (policy, identity)}, so sources
|
||||
* track files in place. Identities are opaque source-owned strings; versions are two-tier: a cheap
|
||||
* gate compared every sweep plus an optional content hash consulted only when the gate moves.
|
||||
* Presence reconciliation ({@link #markSeen} + {@link #deleteUnseen}) keeps the table bounded.
|
||||
*/
|
||||
public interface ProcessedLedger {
|
||||
|
||||
/**
|
||||
* Claims of one version before an {@link ProcessedFileStatus#INTERRUPTED} row stops retrying.
|
||||
*/
|
||||
int MAX_ATTEMPTS = 3;
|
||||
|
||||
/**
|
||||
* One-query snapshot of the rows for these identities, keyed by identity; identities with no
|
||||
* row are absent. Feeds the {@code observed} parameter of {@link #claim(String, String, String,
|
||||
* Supplier, ClaimState)} so a sweep decides its claims without a per-file lookup.
|
||||
*/
|
||||
Map<String, ClaimState> statesFor(String policyId, Collection<String> identities);
|
||||
|
||||
/**
|
||||
* Atomically claim a file at its current version, deciding against {@code observed} (this row's
|
||||
* entry from {@link #statesFor}; null means no row was seen); true means this caller runs it. A
|
||||
* stale {@code observed} cannot double-claim - every transition re-checks the observed state,
|
||||
* so a lost race skips until a later sweep. A null {@code contentHash} makes any gate change a
|
||||
* new version; a non-null supplier is invoked at most once, only on a gate mismatch, and a
|
||||
* matching hash refreshes the stored gate instead of reprocessing. Supplier exceptions
|
||||
* propagate.
|
||||
*/
|
||||
boolean claim(
|
||||
String policyId,
|
||||
String identity,
|
||||
String gate,
|
||||
Supplier<String> contentHash,
|
||||
ClaimState observed);
|
||||
|
||||
/** Snapshot-then-claim convenience for a single file; sweeps batch via {@link #statesFor}. */
|
||||
default boolean claim(
|
||||
String policyId, String identity, String gate, Supplier<String> contentHash) {
|
||||
return claim(
|
||||
policyId,
|
||||
identity,
|
||||
gate,
|
||||
contentHash,
|
||||
statesFor(policyId, List.of(identity)).get(identity));
|
||||
}
|
||||
|
||||
/** Record a claimed file's outcome at its final version ({@code finalContentHash} nullable). */
|
||||
void settle(
|
||||
String policyId,
|
||||
String identity,
|
||||
String finalGate,
|
||||
String finalContentHash,
|
||||
boolean success);
|
||||
|
||||
/**
|
||||
* Record a produced file as {@link ProcessedFileStatus#DONE} so the policy skips its own
|
||||
* outputs. Must be called before the file is visible at this identity; other policies have no
|
||||
* row and still process it.
|
||||
*/
|
||||
void recordOutput(String policyId, String identity, String gate, String contentHash);
|
||||
|
||||
/**
|
||||
* Remove an output record whose file never became visible (its rename lost the name race to a
|
||||
* concurrent writer), so whatever file actually owns that identity is claimable at any version.
|
||||
* A no-op unless the row is still settled exactly as recorded, so a claim that took the row
|
||||
* over in the meantime is left alone.
|
||||
*/
|
||||
void forgetOutput(String policyId, String identity, String gate);
|
||||
|
||||
/**
|
||||
* Whether every row at this identity - across all policies, by design - is {@link
|
||||
* ProcessedFileStatus#DONE}. Consume-mode deletion gates on this so a shared input is removed
|
||||
* only once every claimant has processed it; in-flight, failed, and interrupted rows all veto,
|
||||
* parking the file. Vacuously true when no rows exist.
|
||||
*/
|
||||
boolean allSettledDone(String identity);
|
||||
|
||||
/** Stamp presence for every identity a full-listing sweep observed. */
|
||||
void markSeen(String policyId, Collection<String> identities);
|
||||
|
||||
/**
|
||||
* Remove rows not seen since {@code seenSinceMillis}, keeping in-flight claims. Only call after
|
||||
* every enabled source listed completely; returns the number of rows removed.
|
||||
*/
|
||||
int deleteUnseen(String policyId, long seenSinceMillis);
|
||||
|
||||
/** Forget everything for a policy. */
|
||||
void clearPolicy(String policyId);
|
||||
|
||||
/** Boot recovery: flip stale in-flight claims to {@link ProcessedFileStatus#INTERRUPTED}. */
|
||||
void recoverInterrupted();
|
||||
}
|
||||
+116
-12
@@ -2,11 +2,18 @@ package stirling.software.proprietary.policy.output;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.security.DigestOutputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
@@ -19,14 +26,18 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.proprietary.billing.ContentHasher;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.ledger.FolderIdentities;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
|
||||
/**
|
||||
* Writes a run's outputs to the {@code directory} given in the {@link OutputSpec}. Files are
|
||||
* streamed (not buffered) and uniquely named to avoid clobbering. Returned {@link ResultFile}s
|
||||
* carry a synthetic id since the deliverable is the file on disk, not a {@code FileStorage} entry,
|
||||
* so folder outputs are not downloadable via {@code /files/{id}}.
|
||||
* Writes a run's outputs to the {@code directory} given in the {@link OutputSpec}. Each output is
|
||||
* staged under a hidden {@code .stirling/tmp} dir, recorded in the processed-file ledger, then
|
||||
* atomically renamed into place, so the producing policy's row exists before the file is
|
||||
* discoverable and half-written outputs are never visible. Returned {@link ResultFile}s carry a
|
||||
* synthetic id since the deliverable is the file on disk, not a {@code FileStorage} entry.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -37,7 +48,11 @@ public class FolderOutputSink implements PolicyOutputSink {
|
||||
static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
|
||||
static final String DIRECTORY_OPTION = "directory";
|
||||
|
||||
// Staging entries are renamed away within one delivery; anything older is a crash leftover.
|
||||
private static final Duration STALE_TMP_AGE = Duration.ofDays(1);
|
||||
|
||||
private final FolderAccessGuard accessGuard;
|
||||
private final ProcessedLedger processedLedger;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
@@ -55,20 +70,25 @@ public class FolderOutputSink implements PolicyOutputSink {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResultFile> deliver(String runId, List<Resource> outputs, OutputSpec spec)
|
||||
throws IOException {
|
||||
public List<ResultFile> deliver(
|
||||
OutputDelivery delivery, List<Resource> outputs, OutputSpec spec) throws IOException {
|
||||
Path targetDir = accessGuard.requirePermitted(directoryOf(spec));
|
||||
Files.createDirectories(targetDir);
|
||||
Path canonicalDir = FolderIdentities.canonicalDir(targetDir);
|
||||
Path tmpDir = canonicalDir.resolve(".stirling").resolve("tmp");
|
||||
Files.createDirectories(tmpDir);
|
||||
sweepStaleTmp(tmpDir);
|
||||
|
||||
List<ResultFile> results = new ArrayList<>();
|
||||
for (int i = 0; i < outputs.size(); i++) {
|
||||
Resource resource = outputs.get(i);
|
||||
String name = safeName(resource.getFilename(), i);
|
||||
Path target = uniqueTarget(targetDir, name);
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
Files.copy(is, target);
|
||||
}
|
||||
long size = Files.size(target);
|
||||
Path staged = tmpDir.resolve(UUID.randomUUID().toString());
|
||||
String contentHash = stage(resource, staged, delivery.policyId() != null);
|
||||
long size = Files.size(staged);
|
||||
// Size and mtime survive the rename.
|
||||
String gate = FolderIdentities.statGate(staged);
|
||||
Path target = moveIntoPlace(delivery, canonicalDir, name, staged, gate, contentHash);
|
||||
String contentType =
|
||||
MediaTypeFactory.getMediaType(name)
|
||||
.orElse(MediaType.APPLICATION_OCTET_STREAM)
|
||||
@@ -80,11 +100,95 @@ public class FolderOutputSink implements PolicyOutputSink {
|
||||
.contentType(contentType)
|
||||
.fileSize(size)
|
||||
.build());
|
||||
log.debug("Wrote policy run {} output to {}", runId, target);
|
||||
log.debug("Wrote policy run {} output to {}", delivery.runId(), target);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream the output to its staging path. For a recorded delivery (stored policy) the content
|
||||
* hash is digested in the same pass, so the ledger gets both version tiers without re-reading a
|
||||
* possibly huge output; ad-hoc runs record nothing and skip the digest entirely.
|
||||
*/
|
||||
private static String stage(Resource resource, Path staged, boolean hashed) throws IOException {
|
||||
if (!hashed) {
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
Files.copy(is, staged);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
MessageDigest digest = ContentHasher.newSha256();
|
||||
try (InputStream is = resource.getInputStream();
|
||||
DigestOutputStream out =
|
||||
new DigestOutputStream(Files.newOutputStream(staged), digest)) {
|
||||
is.transferTo(out);
|
||||
}
|
||||
return ContentHasher.toHex(digest.digest());
|
||||
}
|
||||
|
||||
/**
|
||||
* The ledger row must exist before the file is visible at its final path, or a sweep could
|
||||
* claim the producing policy's own output in the gap. Losing the chosen name to a concurrent
|
||||
* writer forgets the just-recorded row - whatever file actually owns that name must stay
|
||||
* claimable at any version - then re-picks.
|
||||
*/
|
||||
private Path moveIntoPlace(
|
||||
OutputDelivery delivery,
|
||||
Path dir,
|
||||
String name,
|
||||
Path staged,
|
||||
String gate,
|
||||
String contentHash)
|
||||
throws IOException {
|
||||
while (true) {
|
||||
Path target = uniqueTarget(dir, name);
|
||||
if (delivery.policyId() != null) {
|
||||
processedLedger.recordOutput(
|
||||
delivery.policyId(), target.toString(), gate, contentHash);
|
||||
}
|
||||
try {
|
||||
Files.move(staged, target, StandardCopyOption.ATOMIC_MOVE);
|
||||
return target;
|
||||
} catch (FileAlreadyExistsException raced) {
|
||||
if (delivery.policyId() != null) {
|
||||
processedLedger.forgetOutput(delivery.policyId(), target.toString(), gate);
|
||||
}
|
||||
log.debug("Output name {} taken concurrently; re-picking", target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort removal of staging leftovers from crashed deliveries. */
|
||||
private static void sweepStaleTmp(Path tmpDir) {
|
||||
Instant cutoff = Instant.now().minus(STALE_TMP_AGE);
|
||||
try (Stream<Path> entries = Files.list(tmpDir)) {
|
||||
entries.filter(Files::isRegularFile)
|
||||
.filter(
|
||||
entry -> {
|
||||
try {
|
||||
return Files.getLastModifiedTime(entry)
|
||||
.toInstant()
|
||||
.isBefore(cutoff);
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.forEach(
|
||||
entry -> {
|
||||
try {
|
||||
Files.deleteIfExists(entry);
|
||||
} catch (IOException e) {
|
||||
log.debug(
|
||||
"Could not remove stale staging file {}: {}",
|
||||
entry,
|
||||
e.getMessage());
|
||||
}
|
||||
});
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not sweep staging dir {}: {}", tmpDir, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static Path directoryOf(OutputSpec spec) {
|
||||
Object directory = spec.options().get(DIRECTORY_OPTION);
|
||||
if (directory == null || directory.toString().isBlank()) {
|
||||
|
||||
+2
-2
@@ -41,8 +41,8 @@ public class InlineOutputSink implements PolicyOutputSink {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResultFile> deliver(String runId, List<Resource> outputs, OutputSpec spec)
|
||||
throws IOException {
|
||||
public List<ResultFile> deliver(
|
||||
OutputDelivery delivery, List<Resource> outputs, OutputSpec spec) throws IOException {
|
||||
List<ResultFile> results = new ArrayList<>();
|
||||
for (int i = 0; i < outputs.size(); i++) {
|
||||
Resource resource = outputs.get(i);
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
package stirling.software.proprietary.policy.output;
|
||||
|
||||
/**
|
||||
* Context for one run's output delivery. {@code policyId} is null for ad-hoc pipelines; when
|
||||
* present, sinks record outputs in the processed-file ledger so the producing policy does not
|
||||
* re-ingest them.
|
||||
*/
|
||||
public record OutputDelivery(String runId, String policyId) {}
|
||||
+1
-1
@@ -25,6 +25,6 @@ public interface PolicyOutputSink {
|
||||
default void validate(OutputSpec spec) {}
|
||||
|
||||
/** Persist/deliver the output files and return their descriptors. */
|
||||
List<ResultFile> deliver(String runId, List<Resource> outputs, OutputSpec spec)
|
||||
List<ResultFile> deliver(OutputDelivery delivery, List<Resource> outputs, OutputSpec spec)
|
||||
throws IOException;
|
||||
}
|
||||
|
||||
+34
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.policy.store;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -16,6 +17,8 @@ import stirling.software.proprietary.policy.model.Policy;
|
||||
public class InProcessPolicyStore implements PolicyStore {
|
||||
|
||||
private final Map<String, Policy> policies = new ConcurrentHashMap<>();
|
||||
// Run-order position per policy id, mirroring JpaPolicyStore's sort_order column.
|
||||
private final Map<String, Integer> sortOrders = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public Policy save(Policy policy) {
|
||||
@@ -35,9 +38,25 @@ public class InProcessPolicyStore implements PolicyStore {
|
||||
policy.output(),
|
||||
policy.teamId());
|
||||
policies.put(id, stored);
|
||||
// Existing policy keeps its position; a new one appends to the end of its team's queue.
|
||||
sortOrders.computeIfAbsent(id, key -> nextSortOrder(stored.teamId()));
|
||||
return stored;
|
||||
}
|
||||
|
||||
private int nextSortOrder(Long teamId) {
|
||||
return policies.values().stream()
|
||||
.filter(policy -> Objects.equals(policy.teamId(), teamId))
|
||||
.map(policy -> sortOrders.getOrDefault(policy.id(), 0))
|
||||
.max(Comparator.naturalOrder())
|
||||
.orElse(-1)
|
||||
+ 1;
|
||||
}
|
||||
|
||||
private Comparator<Policy> byRunOrder() {
|
||||
return Comparator.<Policy>comparingInt(policy -> sortOrders.getOrDefault(policy.id(), 0))
|
||||
.thenComparing(Policy::id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Policy> get(String id) {
|
||||
return Optional.ofNullable(policies.get(id));
|
||||
@@ -45,13 +64,14 @@ public class InProcessPolicyStore implements PolicyStore {
|
||||
|
||||
@Override
|
||||
public List<Policy> all() {
|
||||
return List.copyOf(policies.values());
|
||||
return policies.values().stream().sorted(byRunOrder()).toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Policy> findByTeam(Long teamId) {
|
||||
return policies.values().stream()
|
||||
.filter(policy -> Objects.equals(policy.teamId(), teamId))
|
||||
.sorted(byRunOrder())
|
||||
.toList();
|
||||
}
|
||||
|
||||
@@ -64,8 +84,21 @@ public class InProcessPolicyStore implements PolicyStore {
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reorder(Long teamId, List<String> orderedIds) {
|
||||
int position = 0;
|
||||
for (String id : orderedIds) {
|
||||
Policy policy = policies.get(id);
|
||||
if (policy == null || !Objects.equals(policy.teamId(), teamId)) {
|
||||
continue;
|
||||
}
|
||||
sortOrders.put(id, position++);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(String id) {
|
||||
sortOrders.remove(id);
|
||||
return policies.remove(id) != null;
|
||||
}
|
||||
}
|
||||
|
||||
+42
-1
@@ -1,11 +1,13 @@
|
||||
package stirling.software.proprietary.policy.store;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -26,6 +28,7 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Policy save(Policy policy) {
|
||||
String id =
|
||||
policy.id() == null || policy.id().isBlank()
|
||||
@@ -50,11 +53,49 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
entity.setEnabled(stored.enabled());
|
||||
entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type());
|
||||
entity.setTeamId(stored.teamId());
|
||||
// Preserve an existing policy's run-order position; append a new one to the end of its
|
||||
// team's queue (max + 1), so setting up a policy adds it last by default.
|
||||
entity.setSortOrder(
|
||||
repository
|
||||
.findById(id)
|
||||
.map(PolicyEntity::getSortOrder)
|
||||
.orElseGet(() -> nextSortOrder(stored.teamId())));
|
||||
entity.setPolicyJson(objectMapper.writeValueAsString(stored));
|
||||
repository.save(entity);
|
||||
return stored;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append position for a new policy: max(existing) + 1, computed under a pessimistic lock on the
|
||||
* team's rows (see {@link PolicyRepository#findByTeamForUpdate}) so two concurrent creates
|
||||
* can't both read the same max and assign a duplicate order. (A brand-new team has no rows to
|
||||
* lock; a rare simultaneous first-create there ties at 0 — harmless, since the ordering query
|
||||
* breaks ties by id and any later reorder normalises it.)
|
||||
*/
|
||||
private int nextSortOrder(Long teamId) {
|
||||
return repository.findByTeamForUpdate(teamId).stream()
|
||||
.map(entity -> entity.getSortOrder() == null ? 0 : entity.getSortOrder())
|
||||
.max(Integer::compareTo)
|
||||
.orElse(-1)
|
||||
+ 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public void reorder(Long teamId, List<String> orderedIds) {
|
||||
int position = 0;
|
||||
for (String id : orderedIds) {
|
||||
PolicyEntity entity = repository.findById(id).orElse(null);
|
||||
// Ignore unknown ids and any policy outside the caller's team — a reorder can't reach
|
||||
// across teams.
|
||||
if (entity == null || !Objects.equals(entity.getTeamId(), teamId)) {
|
||||
continue;
|
||||
}
|
||||
entity.setSortOrder(position++);
|
||||
repository.save(entity);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Policy> get(String id) {
|
||||
return repository.findById(id).map(this::toPolicy);
|
||||
@@ -62,7 +103,7 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
|
||||
@Override
|
||||
public List<Policy> all() {
|
||||
return repository.findAll().stream().map(this::toPolicy).toList();
|
||||
return repository.findAllOrdered().stream().map(this::toPolicy).toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+8
@@ -47,6 +47,14 @@ public class PolicyEntity implements Serializable {
|
||||
@Column(name = "team_id")
|
||||
private Long teamId;
|
||||
|
||||
/**
|
||||
* Position in the team's run order (ascending). Team-wide and admin-editable; the per-trigger
|
||||
* order the UI shows is this single sequence filtered by trigger. New policies are appended
|
||||
* (max + 1). Nullable for pre-existing rows; treated as 0 when sorting.
|
||||
*/
|
||||
@Column(name = "sort_order")
|
||||
private Integer sortOrder;
|
||||
|
||||
@Column(name = "policy_json", columnDefinition = "text")
|
||||
private String policyJson;
|
||||
}
|
||||
|
||||
+24
-4
@@ -3,10 +3,13 @@ package stirling.software.proprietary.policy.store;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Lock;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import jakarta.persistence.LockModeType;
|
||||
|
||||
@Repository
|
||||
public interface PolicyRepository extends JpaRepository<PolicyEntity, String> {
|
||||
|
||||
@@ -14,12 +17,29 @@ public interface PolicyRepository extends JpaRepository<PolicyEntity, String> {
|
||||
List<PolicyEntity> findByTriggerTypeAndEnabledTrue(String triggerType);
|
||||
|
||||
/**
|
||||
* Policies belonging to a team, loaded without scanning every team's rows. A {@code null}
|
||||
* teamId matches the rows with no team (login-disabled / pre-team data), mirroring the
|
||||
* in-memory team filter rather than the empty result a plain {@code = null} would give.
|
||||
* Policies belonging to a team, in run order (ascending {@code sortOrder}; a null order sorts
|
||||
* first, id breaks ties for stability). A {@code null} teamId matches the rows with no team
|
||||
* (login-disabled / pre-team data), mirroring the in-memory team filter rather than the empty
|
||||
* result a plain {@code = null} would give.
|
||||
*/
|
||||
@Query(
|
||||
"select p from PolicyEntity p where ((:teamId is null and p.teamId is null) or"
|
||||
+ " p.teamId = :teamId) order by coalesce(p.sortOrder, 0) asc, p.id asc")
|
||||
List<PolicyEntity> findByTeam(@Param("teamId") Long teamId);
|
||||
|
||||
/** All policies in run order — used when team scoping is off (login-disabled). */
|
||||
@Query("select p from PolicyEntity p order by coalesce(p.sortOrder, 0) asc, p.id asc")
|
||||
List<PolicyEntity> findAllOrdered();
|
||||
|
||||
/**
|
||||
* The team's policy rows, locked for the transaction (SELECT … FOR UPDATE). Appending a new
|
||||
* policy reads the max {@code sortOrder} from these under the lock, so two concurrent creates
|
||||
* serialize instead of both reading a stale max and assigning the same position. Must be called
|
||||
* inside a transaction.
|
||||
*/
|
||||
@Lock(LockModeType.PESSIMISTIC_WRITE)
|
||||
@Query(
|
||||
"select p from PolicyEntity p where (:teamId is null and p.teamId is null) or"
|
||||
+ " p.teamId = :teamId")
|
||||
List<PolicyEntity> findByTeam(@Param("teamId") Long teamId);
|
||||
List<PolicyEntity> findByTeamForUpdate(@Param("teamId") Long teamId);
|
||||
}
|
||||
|
||||
+7
@@ -21,6 +21,13 @@ public interface PolicyStore {
|
||||
/** Enabled policies with the given trigger type, for background triggers. */
|
||||
List<Policy> findByTriggerType(String triggerType);
|
||||
|
||||
/**
|
||||
* Set the team's run order from {@code orderedIds} (position → sortOrder). Only policies that
|
||||
* belong to {@code teamId} are touched; unknown/other-team ids are ignored, so a caller can't
|
||||
* reorder across teams. Ids omitted from the list keep their existing order.
|
||||
*/
|
||||
void reorder(Long teamId, List<String> orderedIds);
|
||||
|
||||
/** Returns whether the policy existed. */
|
||||
boolean delete(String id);
|
||||
}
|
||||
|
||||
+3
-1
@@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
@@ -202,7 +203,8 @@ public class FolderWatchTrigger implements PolicyTrigger {
|
||||
}
|
||||
if (dirs.stream().anyMatch(changedDirs::contains)) {
|
||||
log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name());
|
||||
policyRunner.run(policy);
|
||||
// Light: the periodic reconcile does the full sweep.
|
||||
policyRunner.run(policy, SweepKind.LIGHT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
@@ -22,6 +22,7 @@ import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.DatabaseServiceInterface;
|
||||
import stirling.software.proprietary.security.service.SaveUserRequest;
|
||||
import stirling.software.proprietary.security.service.TeamMembershipService;
|
||||
import stirling.software.proprietary.security.service.TeamService;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
import stirling.software.proprietary.service.UserLicenseSettingsService;
|
||||
@@ -40,6 +41,7 @@ public class InitialSecuritySetup {
|
||||
private final DatabaseServiceInterface databaseService;
|
||||
private final UserLicenseSettingsService licenseSettingsService;
|
||||
private final Environment environment;
|
||||
private final TeamMembershipService teamMembershipService;
|
||||
|
||||
/**
|
||||
* SaaS manages identity in Supabase and billing via PAYG, so the self-host bootstrap steps that
|
||||
@@ -114,6 +116,7 @@ public class InitialSecuritySetup {
|
||||
}
|
||||
|
||||
userService.saveAll(usersWithoutTeam); // batch save
|
||||
usersWithoutTeam.forEach(teamMembershipService::syncMembership);
|
||||
if (usersWithoutTeam != null && !usersWithoutTeam.isEmpty()) {
|
||||
log.info(
|
||||
"Assigned {} user(s) without a team to the default team.",
|
||||
|
||||
+6
-2
@@ -34,9 +34,11 @@ import stirling.software.common.model.exception.UnsupportedProviderException;
|
||||
"stirling.software.proprietary.workflow.repository",
|
||||
"stirling.software.proprietary.policy.store",
|
||||
"stirling.software.proprietary.policy.source",
|
||||
"stirling.software.proprietary.policy.ledger",
|
||||
"stirling.software.proprietary.accountlink",
|
||||
"stirling.software.proprietary.access.repository",
|
||||
"stirling.software.proprietary.integration.repository"
|
||||
"stirling.software.proprietary.integration.repository",
|
||||
"stirling.software.proprietary.classification.store"
|
||||
})
|
||||
@EntityScan({
|
||||
"stirling.software.proprietary.security.model",
|
||||
@@ -45,9 +47,11 @@ import stirling.software.common.model.exception.UnsupportedProviderException;
|
||||
"stirling.software.proprietary.workflow.model",
|
||||
"stirling.software.proprietary.policy.store",
|
||||
"stirling.software.proprietary.policy.source",
|
||||
"stirling.software.proprietary.policy.ledger",
|
||||
"stirling.software.proprietary.accountlink",
|
||||
"stirling.software.proprietary.access.model",
|
||||
"stirling.software.proprietary.integration.model"
|
||||
"stirling.software.proprietary.integration.model",
|
||||
"stirling.software.proprietary.classification.store"
|
||||
})
|
||||
public class DatabaseConfig {
|
||||
|
||||
|
||||
+13
-2
@@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.constants.JwtConstants;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.access.service.TeamLeadLookup;
|
||||
import stirling.software.proprietary.audit.AuditEventType;
|
||||
import stirling.software.proprietary.audit.AuditLevel;
|
||||
import stirling.software.proprietary.audit.Audited;
|
||||
@@ -66,6 +67,7 @@ public class AuthController {
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final AiUserDataService aiUserDataService;
|
||||
private final ResourceAccessService resourceAccessService;
|
||||
private final TeamLeadLookup teamLeadLookup;
|
||||
|
||||
/**
|
||||
* Login endpoint - replaces Supabase signInWithPassword
|
||||
@@ -265,8 +267,11 @@ public class AuthController {
|
||||
.body(Map.of("error", "Not authenticated"));
|
||||
}
|
||||
|
||||
UserDetails userDetails = (UserDetails) auth.getPrincipal();
|
||||
User user = (User) userDetails;
|
||||
// Anonymous SaaS sessions carry a raw Jwt principal; treat them as unauthenticated
|
||||
if (!(auth.getPrincipal() instanceof User user)) {
|
||||
return ResponseEntity.status(HttpStatus.UNAUTHORIZED)
|
||||
.body(Map.of("error", "Not authenticated"));
|
||||
}
|
||||
|
||||
return ResponseEntity.ok(Map.of("user", buildUserResponse(user)));
|
||||
|
||||
@@ -631,6 +636,12 @@ public class AuthController {
|
||||
userMap.put("role", user.getRolesAsString());
|
||||
userMap.put("enabled", user.isEnabled());
|
||||
userMap.put("portalAccess", resourceAccessService.canAccessPortal(user));
|
||||
userMap.put("teamLead", teamLeadLookup.isAnyTeamLeader(user));
|
||||
// Expose the caller's team so non-admin team owners can scope their own team's resources.
|
||||
if (user.getTeam() != null) {
|
||||
userMap.put(
|
||||
"team", Map.of("id", user.getTeam().getId(), "name", user.getTeam().getName()));
|
||||
}
|
||||
userMap.put(
|
||||
"authenticationType",
|
||||
user.getAuthenticationType()); // Expose authentication type for SSO detection
|
||||
|
||||
+70
@@ -14,11 +14,15 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.annotations.api.TeamApi;
|
||||
import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.repository.ResourceGrantRepository;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.config.PremiumEndpoint;
|
||||
import stirling.software.proprietary.security.database.repository.UserRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
import stirling.software.proprietary.security.service.TeamMembershipService;
|
||||
import stirling.software.proprietary.security.service.TeamService;
|
||||
|
||||
@TeamApi
|
||||
@@ -29,6 +33,9 @@ public class TeamController {
|
||||
|
||||
private final TeamRepository teamRepository;
|
||||
private final UserRepository userRepository;
|
||||
private final ResourceGrantRepository resourceGrantRepository;
|
||||
private final IntegrationConfigRepository integrationConfigRepository;
|
||||
private final TeamMembershipService teamMembershipService;
|
||||
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@PostMapping("/create")
|
||||
@@ -96,10 +103,72 @@ public class TeamController {
|
||||
"Team must be empty before deletion. Please remove all members first."));
|
||||
}
|
||||
|
||||
if (integrationConfigRepository.existsByOwnerTeam_Id(teamId)) {
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT)
|
||||
.body(
|
||||
Map.of(
|
||||
"error",
|
||||
"Team still owns integration configurations. Delete or reassign them first."));
|
||||
}
|
||||
|
||||
// Team grants and membership rows would dangle once the team row is gone
|
||||
resourceGrantRepository.deleteByPrincipalTypeAndPrincipalId(PrincipalType.TEAM, teamId);
|
||||
teamMembershipService.deleteAllForTeam(teamId);
|
||||
teamRepository.delete(team);
|
||||
return ResponseEntity.ok(Map.of("message", "Team deleted successfully"));
|
||||
}
|
||||
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@PostMapping("/setOwner")
|
||||
@Transactional
|
||||
public ResponseEntity<?> setTeamOwner(
|
||||
@RequestParam("teamId") Long teamId, @RequestParam("userId") Long userId) {
|
||||
return mutateOwner(teamId, userId, true);
|
||||
}
|
||||
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@PostMapping("/removeOwner")
|
||||
@Transactional
|
||||
public ResponseEntity<?> removeTeamOwner(
|
||||
@RequestParam("teamId") Long teamId, @RequestParam("userId") Long userId) {
|
||||
return mutateOwner(teamId, userId, false);
|
||||
}
|
||||
|
||||
private ResponseEntity<?> mutateOwner(Long teamId, Long userId, boolean owner) {
|
||||
Optional<Team> teamOpt = teamRepository.findById(teamId);
|
||||
if (teamOpt.isEmpty()) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND)
|
||||
.body(Map.of("error", "Team not found."));
|
||||
}
|
||||
Team team = teamOpt.get();
|
||||
|
||||
// System teams have no owners
|
||||
if (TeamService.INTERNAL_TEAM_NAME.equals(team.getName())
|
||||
|| TeamService.DEFAULT_TEAM_NAME.equals(team.getName())) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
|
||||
.body(Map.of("error", "System teams cannot have owners."));
|
||||
}
|
||||
|
||||
Optional<User> userOpt = userRepository.findById(userId);
|
||||
if (userOpt.isEmpty()) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND)
|
||||
.body(Map.of("error", "User not found."));
|
||||
}
|
||||
User user = userOpt.get();
|
||||
|
||||
if (user.getTeam() == null || !user.getTeam().getId().equals(teamId)) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
|
||||
.body(Map.of("error", "User must be a member of the team."));
|
||||
}
|
||||
|
||||
if (owner) {
|
||||
teamMembershipService.setOwner(team, user);
|
||||
return ResponseEntity.ok(Map.of("message", "Team owner assigned successfully"));
|
||||
}
|
||||
teamMembershipService.removeOwner(team, user);
|
||||
return ResponseEntity.ok(Map.of("message", "Team owner removed successfully"));
|
||||
}
|
||||
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@PostMapping("/addUser")
|
||||
@Transactional
|
||||
@@ -138,6 +207,7 @@ public class TeamController {
|
||||
// Assign user to team
|
||||
user.setTeam(team);
|
||||
userRepository.save(user);
|
||||
teamMembershipService.syncMembership(user);
|
||||
|
||||
return ResponseEntity.ok(Map.of("message", "User added to team successfully"));
|
||||
}
|
||||
|
||||
+3
@@ -50,6 +50,7 @@ import stirling.software.proprietary.security.saml2.CustomSaml2AuthenticatedPrin
|
||||
import stirling.software.proprietary.security.service.EmailService;
|
||||
import stirling.software.proprietary.security.service.LoginAttemptService;
|
||||
import stirling.software.proprietary.security.service.SaveUserRequest;
|
||||
import stirling.software.proprietary.security.service.TeamMembershipService;
|
||||
import stirling.software.proprietary.security.service.TeamService;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
import stirling.software.proprietary.security.session.SessionPersistentRegistry;
|
||||
@@ -69,6 +70,7 @@ public class UserController {
|
||||
private final Optional<EmailService> emailService;
|
||||
private final UserLicenseSettingsService licenseSettingsService;
|
||||
private final LoginAttemptService loginAttemptService;
|
||||
private final TeamMembershipService teamMembershipService;
|
||||
|
||||
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
|
||||
@PostMapping("/register")
|
||||
@@ -644,6 +646,7 @@ public class UserController {
|
||||
|
||||
user.setTeam(team);
|
||||
userRepository.save(user);
|
||||
teamMembershipService.syncMembership(user);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+8
@@ -50,6 +50,14 @@ public class AdminUserSummary {
|
||||
@Schema(description = "Team membership (if any)")
|
||||
private TeamSummary team;
|
||||
|
||||
@Schema(description = "Whether the user owns (leads) any team")
|
||||
private boolean teamLead;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Whether the user may access the portal, per the server-side access policy")
|
||||
private boolean portalAccess;
|
||||
|
||||
@Schema(description = "User account creation timestamp")
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
|
||||
+24
-2
@@ -1,15 +1,17 @@
|
||||
package stirling.software.saas.repository;
|
||||
package stirling.software.proprietary.security.repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.saas.model.TeamMembership;
|
||||
import stirling.software.proprietary.model.TeamMembership;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
@Repository
|
||||
public interface TeamMembershipRepository extends JpaRepository<TeamMembership, Long> {
|
||||
@@ -106,4 +108,24 @@ public interface TeamMembershipRepository extends JpaRepository<TeamMembership,
|
||||
* @param userId the user ID
|
||||
*/
|
||||
void deleteByTeamIdAndUserId(Long teamId, Long userId);
|
||||
|
||||
/** Leadership checks for TeamLeadLookup. */
|
||||
boolean existsByTeamIdAndUserIdAndRole(Long teamId, Long userId, TeamRole role);
|
||||
|
||||
boolean existsByUserIdAndRole(Long userId, TeamRole role);
|
||||
|
||||
/** All rows holding a role, users and teams pre-fetched for out-of-session mapping. */
|
||||
@Query(
|
||||
"SELECT tm FROM TeamMembership tm JOIN FETCH tm.user JOIN FETCH tm.team"
|
||||
+ " WHERE tm.role = :role")
|
||||
List<TeamMembership> findByRoleFetchingUserAndTeam(@Param("role") TeamRole role);
|
||||
|
||||
void deleteByTeamId(Long teamId);
|
||||
|
||||
void deleteByUserId(Long userId);
|
||||
|
||||
// Detach invitation references so deleting the inviting user does not hit the FK.
|
||||
@Modifying
|
||||
@Query("update TeamMembership tm set tm.invitedBy = null where tm.invitedBy = :user")
|
||||
void clearInvitedBy(@Param("user") User user);
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package stirling.software.proprietary.security.service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.model.TeamMembership;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamMembershipRepository;
|
||||
|
||||
/**
|
||||
* Keeps team_memberships in step with users.team_id on self-hosted admin flows and holds the
|
||||
* team-owner (LEADER role) mutations. SaaS owns membership lifecycle (seats, personal teams,
|
||||
* last-leader guards) via SaasTeamService, so these mutations no-op on the saas profile to avoid
|
||||
* corrupting its accounting.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TeamMembershipService {
|
||||
|
||||
private final TeamMembershipRepository membershipRepository;
|
||||
private final Environment environment;
|
||||
|
||||
private boolean isSaas() {
|
||||
return Arrays.asList(environment.getActiveProfiles()).contains("saas");
|
||||
}
|
||||
|
||||
/** Reflects users.team_id into membership rows, preserving an existing role on the team. */
|
||||
@Transactional
|
||||
public void syncMembership(User user) {
|
||||
if (isSaas() || user == null || user.getId() == null) {
|
||||
return;
|
||||
}
|
||||
Long teamId = user.getTeam() != null ? user.getTeam().getId() : null;
|
||||
boolean present = false;
|
||||
for (TeamMembership row : membershipRepository.findByUserId(user.getId())) {
|
||||
if (teamId != null && teamId.equals(row.getTeam().getId())) {
|
||||
present = true;
|
||||
} else {
|
||||
membershipRepository.delete(row);
|
||||
}
|
||||
}
|
||||
if (teamId != null && !present) {
|
||||
membershipRepository.save(newRow(user.getTeam(), user, TeamRole.MEMBER));
|
||||
}
|
||||
}
|
||||
|
||||
/** Promotes a member to team owner, creating the membership row if it is missing. */
|
||||
@Transactional
|
||||
public void setOwner(Team team, User user) {
|
||||
if (isSaas()) {
|
||||
return;
|
||||
}
|
||||
Optional<TeamMembership> existing =
|
||||
membershipRepository.findByTeamIdAndUserId(team.getId(), user.getId());
|
||||
if (existing.isPresent()) {
|
||||
existing.get().setRole(TeamRole.LEADER);
|
||||
membershipRepository.save(existing.get());
|
||||
} else {
|
||||
membershipRepository.save(newRow(team, user, TeamRole.LEADER));
|
||||
}
|
||||
}
|
||||
|
||||
/** Demotes a team owner back to member; keeps the membership row. */
|
||||
@Transactional
|
||||
public void removeOwner(Team team, User user) {
|
||||
if (isSaas()) {
|
||||
return;
|
||||
}
|
||||
membershipRepository
|
||||
.findByTeamIdAndUserId(team.getId(), user.getId())
|
||||
.ifPresent(
|
||||
row -> {
|
||||
row.setRole(TeamRole.MEMBER);
|
||||
membershipRepository.save(row);
|
||||
});
|
||||
}
|
||||
|
||||
/** Owner user ids for a team. */
|
||||
@Transactional(readOnly = true)
|
||||
public List<Long> ownerUserIds(Long teamId) {
|
||||
return membershipRepository.findByTeamIdAndRole(teamId, TeamRole.LEADER).stream()
|
||||
.map(row -> row.getUser().getId())
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteAllForTeam(Long teamId) {
|
||||
membershipRepository.deleteByTeamId(teamId);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteAllForUser(User user) {
|
||||
membershipRepository.deleteByUserId(user.getId());
|
||||
membershipRepository.clearInvitedBy(user);
|
||||
}
|
||||
|
||||
private TeamMembership newRow(Team team, User user, TeamRole role) {
|
||||
TeamMembership row = new TeamMembership();
|
||||
row.setTeam(team);
|
||||
row.setUser(user);
|
||||
row.setRole(role);
|
||||
// Self-hosted rows are admin-assigned, not invitation-driven.
|
||||
row.setInvitedAt(LocalDateTime.now());
|
||||
row.setAcceptedAt(LocalDateTime.now());
|
||||
return row;
|
||||
}
|
||||
}
|
||||
+25
@@ -39,6 +39,11 @@ import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.common.model.exception.UnsupportedProviderException;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.access.repository.ResourceGrantRepository;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.database.repository.AuthorityRepository;
|
||||
import stirling.software.proprietary.security.database.repository.PersistentLoginRepository;
|
||||
@@ -87,6 +92,9 @@ public class UserService implements UserServiceInterface {
|
||||
private final StorageCleanupEntryRepository storageCleanupEntryRepository;
|
||||
private final FileShareRepository fileShareRepository;
|
||||
private final FileShareAccessRepository fileShareAccessRepository;
|
||||
private final ResourceGrantRepository resourceGrantRepository;
|
||||
private final IntegrationConfigRepository integrationConfigRepository;
|
||||
private final TeamMembershipService teamMembershipService;
|
||||
|
||||
@Transactional
|
||||
public void processSSOPostLogin(
|
||||
@@ -249,6 +257,21 @@ public class UserService implements UserServiceInterface {
|
||||
private void deleteUserRelatedData(User user) {
|
||||
log.info("Deleting all associated data for user: {}", user.getUsername());
|
||||
|
||||
// Drop ACL grants held by this user and detach grants they issued
|
||||
resourceGrantRepository.deleteByPrincipalTypeAndPrincipalId(
|
||||
PrincipalType.USER, user.getId());
|
||||
resourceGrantRepository.clearGrantedBy(user);
|
||||
|
||||
// Integration configs owned by this user FK the users row; drop them and their grants
|
||||
for (IntegrationConfig cfg : integrationConfigRepository.findByOwnerUser(user)) {
|
||||
resourceGrantRepository.deleteByResourceTypeAndResourceId(
|
||||
ResourceType.INTEGRATION_CONFIG, String.valueOf(cfg.getId()));
|
||||
}
|
||||
integrationConfigRepository.deleteByOwnerUser(user);
|
||||
|
||||
// Membership rows and invitation references would dangle once the user row is gone
|
||||
teamMembershipService.deleteAllForUser(user);
|
||||
|
||||
// Delete server certificate (non-nullable OneToOne → User)
|
||||
userServerCertificateService.deleteUserCertificate(user.getId());
|
||||
|
||||
@@ -412,6 +435,7 @@ public class UserService implements UserServiceInterface {
|
||||
}
|
||||
user.setTeam(team);
|
||||
userRepository.save(user);
|
||||
teamMembershipService.syncMembership(user);
|
||||
databaseService.exportDatabase();
|
||||
}
|
||||
|
||||
@@ -523,6 +547,7 @@ public class UserService implements UserServiceInterface {
|
||||
|
||||
// Save user
|
||||
userRepository.save(user);
|
||||
teamMembershipService.syncMembership(user);
|
||||
|
||||
// Export database
|
||||
databaseService.exportDatabase();
|
||||
|
||||
+33
-7
@@ -404,6 +404,7 @@ public class AiWorkflowService {
|
||||
buildCompletedResponse(
|
||||
response.getRationale(),
|
||||
result.files(),
|
||||
result.origins(),
|
||||
inputFileNames(filesById),
|
||||
result.report()));
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
@@ -465,7 +466,8 @@ public class AiWorkflowService {
|
||||
}
|
||||
};
|
||||
return new WorkflowState.Terminal(
|
||||
buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null));
|
||||
buildCompletedResponse(
|
||||
response.getSummary(), List.of(resource), null, List.of(), null));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@@ -523,7 +525,11 @@ public class AiWorkflowService {
|
||||
|
||||
return new WorkflowState.Terminal(
|
||||
buildCompletedResponse(
|
||||
summary, result.files(), inputFileNames(filesById), result.report()));
|
||||
summary,
|
||||
result.files(),
|
||||
result.origins(),
|
||||
inputFileNames(filesById),
|
||||
result.report()));
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage());
|
||||
return new WorkflowState.Terminal(
|
||||
@@ -612,19 +618,35 @@ public class AiWorkflowService {
|
||||
private AiWorkflowResponse buildCompletedResponse(
|
||||
String summary,
|
||||
List<Resource> resultFiles,
|
||||
List<Integer> origins,
|
||||
List<String> inputFileNames,
|
||||
JsonNode report)
|
||||
throws IOException {
|
||||
// Store every output file individually so each gets its own Stirling file ID and the
|
||||
// frontend can add them as independent variants without going through a zip.
|
||||
boolean preserveInputNames = inputFileNames.size() == resultFiles.size();
|
||||
// Count outputs per source so only a clean 1:1 transform (one output for a source) reuses
|
||||
// the input's name; a split (one input → many outputs) keeps each entry's own name.
|
||||
Map<Integer, Long> outputsPerOrigin =
|
||||
origins == null
|
||||
? Map.of()
|
||||
: origins.stream()
|
||||
.filter(o -> o != null)
|
||||
.collect(Collectors.groupingBy(o -> o, Collectors.counting()));
|
||||
List<AiWorkflowResultFile> descriptors = new ArrayList<>();
|
||||
for (int i = 0; i < resultFiles.size(); i++) {
|
||||
Resource resource = resultFiles.get(i);
|
||||
String responseName = resource.getFilename();
|
||||
String inputName = preserveInputNames ? inputFileNames.get(i) : null;
|
||||
// Prefer the input name only for 1:1 operations where the output keeps the same
|
||||
// extension (rotate, compress, etc.). For converters and other extension-changing
|
||||
// The output's source input (from the executor), used both to name it and to tell the
|
||||
// client which file to version in place.
|
||||
Integer origin = origins != null && i < origins.size() ? origins.get(i) : null;
|
||||
boolean uniqueOrigin =
|
||||
origin != null && outputsPerOrigin.getOrDefault(origin, 0L) == 1L;
|
||||
String inputName =
|
||||
uniqueOrigin && origin >= 0 && origin < inputFileNames.size()
|
||||
? inputFileNames.get(origin)
|
||||
: null;
|
||||
// Prefer the source input's name only for 1:1 operations where the output keeps the
|
||||
// same extension (rotate, compress, etc.). For converters and other extension-changing
|
||||
// tools, the response filename from Content-Disposition is authoritative.
|
||||
String name;
|
||||
if (inputName != null
|
||||
@@ -644,7 +666,11 @@ public class AiWorkflowService {
|
||||
try (java.io.InputStream is = resource.getInputStream()) {
|
||||
fileId = fileStorage.storeInputStream(is, name).fileId();
|
||||
}
|
||||
descriptors.add(new AiWorkflowResultFile(fileId, name, contentType));
|
||||
// Only expose the source when this is a clean 1:1 transform, so the client can treat a
|
||||
// present sourceIndex as "replace that input in place" without further disambiguation.
|
||||
descriptors.add(
|
||||
new AiWorkflowResultFile(
|
||||
fileId, name, contentType, uniqueOrigin ? origin : null));
|
||||
}
|
||||
|
||||
AiWorkflowResponse completed = new AiWorkflowResponse();
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package stirling.software.proprietary.access.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ResourceAccessSecurityTest {
|
||||
|
||||
@Mock private ResourceAccessService accessService;
|
||||
@Mock private UserService userService;
|
||||
|
||||
@InjectMocks private ResourceAccessSecurity security;
|
||||
|
||||
@AfterEach
|
||||
void clearContext() {
|
||||
SecurityContextHolder.clearContext();
|
||||
}
|
||||
|
||||
@Test
|
||||
void userPrincipalDelegatesToPortalCheck() {
|
||||
User user = new User();
|
||||
user.setId(5L);
|
||||
authenticate(user);
|
||||
when(accessService.canAccessPortal(user)).thenReturn(true);
|
||||
|
||||
assertThat(security.canUsePortal()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void deniedWithoutAuthentication() {
|
||||
assertThat(security.canUsePortal()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonUserPrincipalWithoutBackingRowIsDenied() {
|
||||
// Mirrors an anonymous SaaS session: principal is not a User and resolves to nothing.
|
||||
authenticate("anonymousUser");
|
||||
|
||||
assertThat(security.canUsePortal()).isFalse();
|
||||
}
|
||||
|
||||
private void authenticate(Object principal) {
|
||||
SecurityContextHolder.getContext()
|
||||
.setAuthentication(
|
||||
new UsernamePasswordAuthenticationToken(
|
||||
principal, null, java.util.List.of()));
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package stirling.software.proprietary.access.service;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.access.model.PrincipalRef;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
class DefaultPrincipalResolverTest {
|
||||
|
||||
private final DefaultPrincipalResolver resolver = new DefaultPrincipalResolver();
|
||||
|
||||
@Test
|
||||
void userWithoutTeamProjectsUser() {
|
||||
assertThat(resolver.principalsOf(user(5, null)))
|
||||
.containsExactlyInAnyOrder(PrincipalRef.user(5L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void userWithTeamProjectsUserAndTeam() {
|
||||
assertThat(resolver.principalsOf(user(5, 7L)))
|
||||
.containsExactlyInAnyOrder(PrincipalRef.user(5L), PrincipalRef.team(7L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullUserProjectsNothing() {
|
||||
assertThat(resolver.principalsOf(null)).isEmpty();
|
||||
assertThat(resolver.principalTokens(null)).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void tokensUseTheCanonicalWireForm() {
|
||||
assertThat(resolver.principalTokens(user(5, 7L)))
|
||||
.containsExactlyInAnyOrder("user:5", "team:7");
|
||||
}
|
||||
|
||||
@Test
|
||||
void selfHostedAllowsDeploymentWideAccess() {
|
||||
assertThat(resolver.allowsDeploymentWideAccess()).isTrue();
|
||||
}
|
||||
|
||||
private User user(long id, Long teamId) {
|
||||
User u = new User();
|
||||
u.setId(id);
|
||||
if (teamId != null) {
|
||||
Team t = new Team();
|
||||
t.setId(teamId);
|
||||
u.setTeam(t);
|
||||
}
|
||||
return u;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package stirling.software.proprietary.access.service;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamMembershipRepository;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class MembershipTeamLeadLookupTest {
|
||||
|
||||
@Mock private TeamMembershipRepository memberships;
|
||||
|
||||
@InjectMocks private MembershipTeamLeadLookup lookup;
|
||||
|
||||
@Test
|
||||
void leaderMembershipMakesTeamLeader() {
|
||||
when(memberships.existsByTeamIdAndUserIdAndRole(7L, 5L, TeamRole.LEADER)).thenReturn(true);
|
||||
assertThat(lookup.isLeaderOfTeam(user(5), 7L)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void memberOnlyIsNotTeamLeader() {
|
||||
when(memberships.existsByTeamIdAndUserIdAndRole(7L, 5L, TeamRole.LEADER)).thenReturn(false);
|
||||
assertThat(lookup.isLeaderOfTeam(user(5), 7L)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anyLeadershipDetectedAcrossTeams() {
|
||||
when(memberships.existsByUserIdAndRole(5L, TeamRole.LEADER)).thenReturn(true);
|
||||
assertThat(lookup.isAnyTeamLeader(user(5))).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullInputsNeverQueryAndDeny() {
|
||||
assertThat(lookup.isAnyTeamLeader(null)).isFalse();
|
||||
assertThat(lookup.isLeaderOfTeam(null, 7L)).isFalse();
|
||||
assertThat(lookup.isLeaderOfTeam(user(5), null)).isFalse();
|
||||
verifyNoInteractions(memberships);
|
||||
}
|
||||
|
||||
private User user(long id) {
|
||||
User u = new User();
|
||||
u.setId(id);
|
||||
return u;
|
||||
}
|
||||
}
|
||||
+22
@@ -3,6 +3,7 @@ package stirling.software.proprietary.access.service;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.Optional;
|
||||
@@ -150,6 +151,27 @@ class OwnershipServiceTest {
|
||||
assertThat(ownership.canUse(TYPE, r, user(8))).isFalse(); // not a lead
|
||||
}
|
||||
|
||||
@Test
|
||||
void disabledResourceManageableOnlyByOwnerOrAdmin() {
|
||||
TestResource r = new TestResource(1L);
|
||||
r.setEnabled(false);
|
||||
r.setOwnerUser(user(7));
|
||||
|
||||
assertThat(ownership.canManage(TYPE, r, user(7))).isTrue(); // owner
|
||||
assertThat(ownership.canManage(TYPE, r, user(8))).isFalse(); // would-be MANAGE grantee
|
||||
assertThat(ownership.canManage(TYPE, r, admin(2))).isTrue(); // admin
|
||||
// Grants are bypassed entirely while disabled.
|
||||
verifyNoInteractions(accessService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledResourceManageDelegatesToTheAcl() {
|
||||
TestResource r = new TestResource(1L);
|
||||
when(accessService.canManageResource(any(), any(), any(), any())).thenReturn(true);
|
||||
|
||||
assertThat(ownership.canManage(TYPE, r, user(7))).isTrue();
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private void assertForbidden(org.assertj.core.api.ThrowableAssert.ThrowingCallable call) {
|
||||
|
||||
+109
-8
@@ -1,20 +1,22 @@
|
||||
package stirling.software.proprietary.access.service;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.proprietary.access.model.AccessPermission;
|
||||
import stirling.software.proprietary.access.model.DefaultAccessPolicy;
|
||||
import stirling.software.proprietary.access.model.PrincipalRef;
|
||||
import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.model.ResourceGrant;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
@@ -32,13 +34,20 @@ class ResourceAccessServiceTest {
|
||||
@Mock private ResourceGrantRepository grantRepository;
|
||||
@Mock private TeamLeadLookup teamLeadLookup;
|
||||
|
||||
@InjectMocks private ResourceAccessService service;
|
||||
private ResourceAccessService service;
|
||||
|
||||
@BeforeEach
|
||||
void setPortalDefault() throws Exception {
|
||||
void setUp() throws Exception {
|
||||
service = newService(new DefaultPrincipalResolver());
|
||||
}
|
||||
|
||||
private ResourceAccessService newService(PrincipalResolver resolver) throws Exception {
|
||||
ResourceAccessService s =
|
||||
new ResourceAccessService(grantRepository, teamLeadLookup, resolver);
|
||||
Field f = ResourceAccessService.class.getDeclaredField("portalDefaultPolicy");
|
||||
f.setAccessible(true);
|
||||
f.set(service, DefaultAccessPolicy.ADMINS_AND_TEAM_LEADS);
|
||||
f.set(s, DefaultAccessPolicy.ADMINS_AND_TEAM_LEADS);
|
||||
return s;
|
||||
}
|
||||
|
||||
// ---- owner / admin short-circuits ----
|
||||
@@ -55,15 +64,56 @@ class ResourceAccessServiceTest {
|
||||
void ownerMayUseEvenWithExplicitOnly() {
|
||||
assertThat(
|
||||
service.canUseResource(
|
||||
TYPE, RID, 5L, DefaultAccessPolicy.EXPLICIT_ONLY, user(5)))
|
||||
TYPE,
|
||||
RID,
|
||||
PrincipalRef.user(5L),
|
||||
DefaultAccessPolicy.EXPLICIT_ONLY,
|
||||
user(5)))
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullUserIsAlwaysDenied() {
|
||||
assertThat(service.canUseResource(TYPE, RID, 5L, DefaultAccessPolicy.ORG_ALL, null))
|
||||
assertThat(
|
||||
service.canUseResource(
|
||||
TYPE,
|
||||
RID,
|
||||
PrincipalRef.user(5L),
|
||||
DefaultAccessPolicy.ORG_ALL,
|
||||
null))
|
||||
.isFalse();
|
||||
assertThat(service.canManageResource(TYPE, RID, PrincipalRef.user(5L), null)).isFalse();
|
||||
}
|
||||
|
||||
// ---- owner refs ----
|
||||
|
||||
@Test
|
||||
void teamOwnerRefAllowsLeaderOfThatTeam() {
|
||||
User leader = userInTeam(5, 7);
|
||||
when(teamLeadLookup.isLeaderOfTeam(leader, 7L)).thenReturn(true);
|
||||
assertThat(
|
||||
service.canUseResource(
|
||||
TYPE,
|
||||
RID,
|
||||
PrincipalRef.team(7L),
|
||||
DefaultAccessPolicy.EXPLICIT_ONLY,
|
||||
leader))
|
||||
.isTrue();
|
||||
assertThat(service.canManageResource(TYPE, RID, PrincipalRef.team(7L), leader)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void teamOwnerRefDeniesPlainTeamMember() {
|
||||
stubGrants();
|
||||
User member = userInTeam(5, 7);
|
||||
assertThat(
|
||||
service.canUseResource(
|
||||
TYPE,
|
||||
RID,
|
||||
PrincipalRef.team(7L),
|
||||
DefaultAccessPolicy.EXPLICIT_ONLY,
|
||||
member))
|
||||
.isFalse();
|
||||
assertThat(service.canManageResource(TYPE, RID, 5L, null)).isFalse();
|
||||
}
|
||||
|
||||
// ---- explicit grants ----
|
||||
@@ -124,15 +174,60 @@ class ResourceAccessServiceTest {
|
||||
assertThat(service.canManageResource(TYPE, RID, null, user(5))).isTrue();
|
||||
}
|
||||
|
||||
// ---- upsert semantics ----
|
||||
|
||||
@Test
|
||||
void grantOverwritesPermissionEvenDowngrading() {
|
||||
// Upsert key is (type, resourceId, principalType, principalId); permission is overwritten.
|
||||
ResourceGrant existing = grant(PrincipalType.USER, 5L, AccessPermission.MANAGE);
|
||||
stubGrants(existing);
|
||||
when(grantRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
ResourceGrant saved =
|
||||
service.grant(TYPE, RID, PrincipalType.USER, 5L, AccessPermission.USE, null);
|
||||
|
||||
assertThat(saved).isSameAs(existing);
|
||||
assertThat(saved.getPermission()).isEqualTo(AccessPermission.USE);
|
||||
}
|
||||
|
||||
// ---- granted resource ids ----
|
||||
|
||||
@Test
|
||||
void grantedResourceIdsCollectsAcrossAllPrincipals() {
|
||||
when(grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId(
|
||||
TYPE, PrincipalType.USER, 5L))
|
||||
.thenReturn(List.of(grantOn("a")));
|
||||
when(grantRepository.findByResourceTypeAndPrincipalTypeAndPrincipalId(
|
||||
TYPE, PrincipalType.TEAM, 7L))
|
||||
.thenReturn(List.of(grantOn("b")));
|
||||
|
||||
assertThat(service.grantedResourceIds(TYPE, userInTeam(5, 7)))
|
||||
.containsExactlyInAnyOrder("a", "b");
|
||||
}
|
||||
|
||||
// ---- default policies ----
|
||||
|
||||
@Test
|
||||
void orgAllDefaultAllowsAnyUser() {
|
||||
void orgAllDefaultAllowsAnyUserWhenDeploymentWide() {
|
||||
// Self-hosted resolver allows deployment-wide access, so ORG_ALL admits anyone.
|
||||
stubGrants();
|
||||
assertThat(service.canUseResource(TYPE, RID, null, DefaultAccessPolicy.ORG_ALL, user(5)))
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void orgAllDefaultDeniedWhenNotDeploymentWide() throws Exception {
|
||||
// Mirrors the saas resolver (USER/TEAM only, no deployment-wide access): ORG_ALL must not
|
||||
// leak a resource to a tenant's users, so an ungranted user is denied.
|
||||
ResourceAccessService tenantScoped =
|
||||
newService(u -> u == null ? Set.of() : Set.of(PrincipalRef.user(u.getId())));
|
||||
stubGrants();
|
||||
assertThat(
|
||||
tenantScoped.canUseResource(
|
||||
TYPE, RID, null, DefaultAccessPolicy.ORG_ALL, user(5)))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void explicitOnlyDefaultDeniesUngrantedUser() {
|
||||
stubGrants();
|
||||
@@ -194,6 +289,12 @@ class ResourceAccessServiceTest {
|
||||
return g;
|
||||
}
|
||||
|
||||
private ResourceGrant grantOn(String resourceId) {
|
||||
ResourceGrant g = grant(PrincipalType.USER, 5L, AccessPermission.USE);
|
||||
g.setResourceId(resourceId);
|
||||
return g;
|
||||
}
|
||||
|
||||
private User user(long id) {
|
||||
User u = new User();
|
||||
u.setId(id);
|
||||
|
||||
+17
@@ -49,6 +49,23 @@ class SecretMaskerTest {
|
||||
assertThat(merged.get("secretKey")).isEqualTo("NEW");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeDropsKeysAbsentFromIncoming() {
|
||||
// PUT/replace semantics: a key removed in the edit is removed from storage.
|
||||
Map<String, Object> stored = new java.util.LinkedHashMap<>();
|
||||
stored.put("bucket", "b");
|
||||
stored.put("endpoint", "https://old");
|
||||
stored.put("secretKey", "REAL");
|
||||
Map<String, Object> incoming = new java.util.LinkedHashMap<>();
|
||||
incoming.put("bucket", "b");
|
||||
incoming.put("secretKey", SecretMasker.MASK);
|
||||
|
||||
Map<String, Object> merged = masker.merge(stored, incoming);
|
||||
|
||||
assertThat(merged).doesNotContainKey("endpoint");
|
||||
assertThat(merged.get("secretKey")).isEqualTo("REAL"); // masked secret retained
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitizeDropsBlankSecretsOnCreate() {
|
||||
Map<String, Object> incoming = new LinkedHashMap<>();
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
package stirling.software.proprietary.classification;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
import stirling.software.proprietary.classification.store.ClassificationLabelStore;
|
||||
import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@DisplayName("ClassificationLabelsController")
|
||||
class ClassificationLabelsControllerTest {
|
||||
|
||||
private static final Long TEAM = 7L;
|
||||
|
||||
@Mock private PolicyManagementAuthority policyManagementAuthority;
|
||||
@Mock private UserServiceInterface userService;
|
||||
|
||||
private ClassificationLabelStore store;
|
||||
private ApplicationProperties applicationProperties;
|
||||
private ClassificationLabelsController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
store = new InProcessClassificationLabelStore();
|
||||
applicationProperties = new ApplicationProperties();
|
||||
controller =
|
||||
new ClassificationLabelsController(
|
||||
store, policyManagementAuthority, applicationProperties, userService);
|
||||
}
|
||||
|
||||
private static ClassificationLabels sample() {
|
||||
return new ClassificationLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
new ClassificationLabel("contract", "Contract", null)));
|
||||
}
|
||||
|
||||
private void loginEnabled(boolean enabled) {
|
||||
applicationProperties.getSecurity().setEnableLogin(enabled);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("GET returns 204 when the team has no labels")
|
||||
void getEmpty() {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
ResponseEntity<ClassificationLabels> response = controller.getTeamLabels();
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT then GET round-trips the team's labels (login disabled)")
|
||||
void saveThenGet() {
|
||||
loginEnabled(false);
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
|
||||
controller.saveTeamLabels(sample());
|
||||
ResponseEntity<ClassificationLabels> got = controller.getTeamLabels();
|
||||
|
||||
assertThat(got.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
assertThat(got.getBody()).isNotNull();
|
||||
assertThat(got.getBody().labels()).hasSize(2);
|
||||
assertThat(got.getBody().labels().getFirst().name()).isEqualTo("Invoice");
|
||||
assertThat(got.getBody().labels().getFirst().icon()).isEqualTo("receipt-long");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT is scoped per team")
|
||||
void perTeam() {
|
||||
loginEnabled(false);
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
controller.saveTeamLabels(sample());
|
||||
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(99L);
|
||||
assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT is rejected for a non-editor when login is enabled")
|
||||
void putForbiddenForNonEditor() {
|
||||
loginEnabled(true);
|
||||
when(policyManagementAuthority.canEditPolicies()).thenReturn(false);
|
||||
|
||||
assertThatThrownBy(() -> controller.saveTeamLabels(sample()))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.hasFieldOrPropertyWithValue("statusCode", HttpStatus.FORBIDDEN);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT rejects an invalid label set with 400")
|
||||
void putInvalid() {
|
||||
loginEnabled(false);
|
||||
ClassificationLabels duplicate =
|
||||
new ClassificationLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", null),
|
||||
new ClassificationLabel("invoice", "Invoice", null)));
|
||||
|
||||
assertThatThrownBy(() -> controller.saveTeamLabels(duplicate))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.hasFieldOrPropertyWithValue("statusCode", HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("DELETE resets the team back to no stored labels")
|
||||
void deleteResets() {
|
||||
loginEnabled(false);
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
controller.saveTeamLabels(sample());
|
||||
|
||||
ResponseEntity<Void> response = controller.resetTeamLabels();
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
}
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
package stirling.software.proprietary.classification.model;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@DisplayName("LabelsValidator")
|
||||
class LabelsValidatorTest {
|
||||
|
||||
private static ClassificationLabels labels(ClassificationLabel... labels) {
|
||||
return new ClassificationLabels(List.of(labels));
|
||||
}
|
||||
|
||||
private static ClassificationLabel label(String name) {
|
||||
return new ClassificationLabel(slug(name), name, null);
|
||||
}
|
||||
|
||||
private static String slug(String name) {
|
||||
return name.trim()
|
||||
.toLowerCase(Locale.ROOT)
|
||||
.replaceAll("[^a-z0-9]+", "-")
|
||||
.replaceAll("(^-|-$)", "");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("accepts a well-formed label set")
|
||||
void acceptsValid() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
label("Contract"));
|
||||
assertThatCode(() -> LabelsValidator.validate(set)).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("accepts an empty label set (reads as: use the default)")
|
||||
void acceptsEmpty() {
|
||||
assertThatCode(() -> LabelsValidator.validate(labels())).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects a null label set")
|
||||
void rejectsNull() {
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(null))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Labels are required");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects duplicate names (distinct ids)")
|
||||
void rejectsDuplicateNames() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice-a", "Invoice", null),
|
||||
new ClassificationLabel("invoice-b", "Invoice", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Duplicate label name");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects duplicate names differing only by case")
|
||||
void rejectsDuplicateNamesCaseInsensitive() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice-a", "Invoice", null),
|
||||
new ClassificationLabel("invoice-b", "INVOICE", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Duplicate label name");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects duplicate ids")
|
||||
void rejectsDuplicateIds() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice", "Invoice", null),
|
||||
new ClassificationLabel("invoice", "Sales invoice", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Duplicate label id");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects a blank name")
|
||||
void rejectsBlankName() {
|
||||
ClassificationLabels set = labels(new ClassificationLabel("blank", " ", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Label name must not be blank");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects a blank id")
|
||||
void rejectsBlankId() {
|
||||
ClassificationLabels set = labels(new ClassificationLabel(" ", "Invoice", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Label id must not be blank");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects an over-long name")
|
||||
void rejectsOverLongName() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel(
|
||||
"x", "x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1), null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("too long");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects an over-long icon (a null icon is fine)")
|
||||
void rejectsOverLongIcon() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel(
|
||||
"invoice",
|
||||
"Invoice",
|
||||
"x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1)));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("icon is too long");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects more labels than the cap")
|
||||
void rejectsTooManyLabels() {
|
||||
List<ClassificationLabel> tooMany =
|
||||
IntStream.rangeClosed(0, LabelsValidator.MAX_LABELS)
|
||||
.mapToObj(i -> label("label" + i))
|
||||
.toList();
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(new ClassificationLabels(tooMany)))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Too many labels");
|
||||
}
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package stirling.software.proprietary.controller.api;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.service.PdfMetadataService;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.service.AiEngineClient;
|
||||
import stirling.software.proprietary.service.PdfContentExtractor;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@MockitoSettings(strictness = Strictness.LENIENT)
|
||||
class ClassifyLabelControllerTest {
|
||||
|
||||
private static final Long TEAM = 7L;
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private PdfContentExtractor pdfContentExtractor;
|
||||
@Mock private PdfMetadataService pdfMetadataService;
|
||||
@Mock private AiEngineClient aiEngineClient;
|
||||
@Mock private PolicyManagementAuthority policyManagementAuthority;
|
||||
|
||||
private final ObjectMapper objectMapper = JsonMapper.builder().build();
|
||||
private InProcessClassificationLabelStore labelStore;
|
||||
private ClassifyLabelController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
labelStore = new InProcessClassificationLabelStore();
|
||||
controller =
|
||||
new ClassifyLabelController(
|
||||
pdfDocumentFactory,
|
||||
tempFileManager,
|
||||
pdfContentExtractor,
|
||||
pdfMetadataService,
|
||||
aiEngineClient,
|
||||
objectMapper,
|
||||
null,
|
||||
labelStore,
|
||||
policyManagementAuthority);
|
||||
}
|
||||
|
||||
private void stubSinglePageDocument() throws Exception {
|
||||
PDDocument document = mock(PDDocument.class);
|
||||
MultipartFile file = mock(MultipartFile.class);
|
||||
when(file.getOriginalFilename()).thenReturn("invoice.pdf");
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class), eq(true))).thenReturn(document);
|
||||
when(document.getNumberOfPages()).thenReturn(1);
|
||||
when(pdfContentExtractor.extractPageTextRaw(document, 1))
|
||||
.thenReturn("Invoice total due 100.00");
|
||||
when(aiEngineClient.post(eq("/api/v1/documents/classify"), anyString(), isNull()))
|
||||
.thenReturn("{\"outcome\":\"classification\",\"labels\":[\"invoice\"]}");
|
||||
|
||||
try {
|
||||
controller.classifyAndLabel(file);
|
||||
} catch (Exception ignored) {
|
||||
// WebResponseUtils.pdfDocToWebResponse needs a real temp file; the engine call and
|
||||
// metadata write we assert on have already happened by the time it runs.
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode sentEngineRequest() throws Exception {
|
||||
ArgumentCaptor<String> body = ArgumentCaptor.forClass(String.class);
|
||||
verify(aiEngineClient).post(eq("/api/v1/documents/classify"), body.capture(), isNull());
|
||||
return objectMapper.readTree(body.getValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyAndLabel_writesClassificationWithoutOutcome() throws Exception {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
labelStore.save(
|
||||
TEAM,
|
||||
new ClassificationLabels(
|
||||
List.of(new ClassificationLabel("invoice", "Invoice", null))),
|
||||
"admin");
|
||||
|
||||
stubSinglePageDocument();
|
||||
|
||||
ArgumentCaptor<String> value = ArgumentCaptor.forClass(String.class);
|
||||
verify(pdfMetadataService)
|
||||
.setClassificationMetadata(any(PDDocument.class), value.capture());
|
||||
|
||||
JsonNode written = objectMapper.readTree(value.getValue());
|
||||
assertThat(written.has("outcome")).isFalse();
|
||||
// The engine returns label ids; they're stored on the document verbatim.
|
||||
assertThat(written.get("labels").get(0).asText()).isEqualTo("invoice");
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyAndLabel_sendsTeamLabelIdsAndNames() throws Exception {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
labelStore.save(
|
||||
TEAM,
|
||||
new ClassificationLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
new ClassificationLabel("contract", "Contract", null),
|
||||
new ClassificationLabel("timesheet", "Timesheet", null))),
|
||||
"admin");
|
||||
|
||||
stubSinglePageDocument();
|
||||
|
||||
JsonNode request = sentEngineRequest();
|
||||
assertThat(request.get("fileName").asText()).isEqualTo("invoice.pdf");
|
||||
JsonNode labels = request.get("labels");
|
||||
assertThat(labels.isArray()).isTrue();
|
||||
assertThat(labels.size()).isEqualTo(3);
|
||||
// Each entry is an {id, name} pair — the model reasons over names, we store ids.
|
||||
assertThat(
|
||||
List.of(
|
||||
labels.get(0).get("id").asText(),
|
||||
labels.get(1).get("id").asText(),
|
||||
labels.get(2).get("id").asText()))
|
||||
.containsExactly("invoice", "contract", "timesheet");
|
||||
assertThat(
|
||||
List.of(
|
||||
labels.get(0).get("name").asText(),
|
||||
labels.get(1).get("name").asText(),
|
||||
labels.get(2).get("name").asText()))
|
||||
.containsExactly("Invoice", "Contract", "Timesheet");
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyAndLabel_skipsClassificationWhenNothingStored() throws Exception {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
|
||||
stubSinglePageDocument();
|
||||
|
||||
// No team labels stored, and the engine holds no default of its own, so the file is passed
|
||||
// through unlabelled: neither the engine nor the metadata write is invoked.
|
||||
verify(aiEngineClient, never()).post(anyString(), anyString(), any());
|
||||
verify(pdfMetadataService, never())
|
||||
.setClassificationMetadata(any(PDDocument.class), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void windowPageNumbers_takesFirstAndLastWithoutOverlap() {
|
||||
assertEquals(List.of(1, 2, 4, 5), ClassifyLabelController.windowPageNumbers(5, 2));
|
||||
assertEquals(List.of(1, 2, 3), ClassifyLabelController.windowPageNumbers(3, 2));
|
||||
// Short docs clamp + dedupe rather than throwing or going out of range.
|
||||
assertEquals(List.of(1, 2), ClassifyLabelController.windowPageNumbers(2, 2));
|
||||
assertEquals(List.of(1), ClassifyLabelController.windowPageNumbers(1, 2));
|
||||
assertEquals(List.of(), ClassifyLabelController.windowPageNumbers(0, 2));
|
||||
}
|
||||
}
|
||||
+7
-1
@@ -24,6 +24,7 @@ import org.springframework.security.oauth2.core.user.OAuth2User;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.config.AuditConfigurationProperties;
|
||||
import stirling.software.proprietary.controller.api.ProprietaryUIDataController.AccountData;
|
||||
import stirling.software.proprietary.controller.api.ProprietaryUIDataController.AdminSettingsData;
|
||||
@@ -39,6 +40,7 @@ import stirling.software.proprietary.security.database.repository.SessionReposit
|
||||
import stirling.software.proprietary.security.database.repository.UserRepository;
|
||||
import stirling.software.proprietary.security.model.Authority;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamMembershipRepository;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
import stirling.software.proprietary.security.saml2.CustomSaml2AuthenticatedPrincipal;
|
||||
import stirling.software.proprietary.security.service.DatabaseServiceInterface;
|
||||
@@ -57,12 +59,14 @@ class ProprietaryUIDataControllerMoreTest {
|
||||
@Mock private SessionPersistentRegistry sessionPersistentRegistry;
|
||||
@Mock private UserRepository userRepository;
|
||||
@Mock private TeamRepository teamRepository;
|
||||
@Mock private TeamMembershipRepository teamMembershipRepository;
|
||||
@Mock private SessionRepository sessionRepository;
|
||||
@Mock private DatabaseServiceInterface databaseService;
|
||||
@Mock private UserLicenseSettingsService licenseSettingsService;
|
||||
@Mock private PersistentAuditEventRepository auditRepository;
|
||||
@Mock private MfaService mfaService;
|
||||
@Mock private LoginAttemptService loginAttemptService;
|
||||
@Mock private ResourceAccessService resourceAccessService;
|
||||
|
||||
private ApplicationProperties applicationProperties;
|
||||
private AuditConfigurationProperties auditConfig;
|
||||
@@ -87,6 +91,7 @@ class ProprietaryUIDataControllerMoreTest {
|
||||
sessionPersistentRegistry,
|
||||
userRepository,
|
||||
teamRepository,
|
||||
teamMembershipRepository,
|
||||
sessionRepository,
|
||||
databaseService,
|
||||
objectMapper,
|
||||
@@ -94,7 +99,8 @@ class ProprietaryUIDataControllerMoreTest {
|
||||
licenseSettingsService,
|
||||
auditRepository,
|
||||
mfaService,
|
||||
loginAttemptService);
|
||||
loginAttemptService,
|
||||
resourceAccessService);
|
||||
}
|
||||
|
||||
private static User normalUser(Long id, String username) {
|
||||
|
||||
+7
-1
@@ -18,6 +18,7 @@ import org.springframework.security.authentication.UsernamePasswordAuthenticatio
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.config.AuditConfigurationProperties;
|
||||
import stirling.software.proprietary.controller.api.ProprietaryUIDataController.AccountData;
|
||||
import stirling.software.proprietary.controller.api.ProprietaryUIDataController.DatabaseData;
|
||||
@@ -27,6 +28,7 @@ import stirling.software.proprietary.security.database.repository.SessionReposit
|
||||
import stirling.software.proprietary.security.database.repository.UserRepository;
|
||||
import stirling.software.proprietary.security.model.Authority;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamMembershipRepository;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
import stirling.software.proprietary.security.service.DatabaseService;
|
||||
import stirling.software.proprietary.security.service.LoginAttemptService;
|
||||
@@ -43,12 +45,14 @@ class ProprietaryUIDataControllerTest {
|
||||
@Mock private SessionPersistentRegistry sessionPersistentRegistry;
|
||||
@Mock private UserRepository userRepository;
|
||||
@Mock private TeamRepository teamRepository;
|
||||
@Mock private TeamMembershipRepository teamMembershipRepository;
|
||||
@Mock private SessionRepository sessionRepository;
|
||||
@Mock private DatabaseService databaseService;
|
||||
@Mock private UserLicenseSettingsService licenseSettingsService;
|
||||
@Mock private PersistentAuditEventRepository auditRepository;
|
||||
@Mock private MfaService mfaService;
|
||||
@Mock private LoginAttemptService loginAttemptService;
|
||||
@Mock private ResourceAccessService resourceAccessService;
|
||||
|
||||
private ApplicationProperties applicationProperties;
|
||||
private AuditConfigurationProperties auditConfig;
|
||||
@@ -75,6 +79,7 @@ class ProprietaryUIDataControllerTest {
|
||||
sessionPersistentRegistry,
|
||||
userRepository,
|
||||
teamRepository,
|
||||
teamMembershipRepository,
|
||||
sessionRepository,
|
||||
databaseService,
|
||||
objectMapper,
|
||||
@@ -82,7 +87,8 @@ class ProprietaryUIDataControllerTest {
|
||||
licenseSettingsService,
|
||||
auditRepository,
|
||||
mfaService,
|
||||
loginAttemptService);
|
||||
loginAttemptService,
|
||||
resourceAccessService);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+59
-2
@@ -49,6 +49,9 @@ class IntegrationConfigServiceTest {
|
||||
@Mock private OwnershipService ownership;
|
||||
@Mock private SecretMasker secretMasker;
|
||||
|
||||
@Mock
|
||||
private stirling.software.proprietary.access.repository.ResourceGrantRepository grantRepository;
|
||||
|
||||
@InjectMocks private IntegrationConfigService service;
|
||||
|
||||
@Test
|
||||
@@ -58,9 +61,9 @@ class IntegrationConfigServiceTest {
|
||||
User user = user(7);
|
||||
|
||||
IntegrationConfig created =
|
||||
service.create(request(IntegrationType.S3, OwnerScope.USER, null), user);
|
||||
service.create(request(IntegrationType.API, OwnerScope.USER, null), user);
|
||||
|
||||
assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.S3);
|
||||
assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.API);
|
||||
assertThat(created.getName()).isEqualTo("name");
|
||||
verify(ownership)
|
||||
.assignOwnership(eq(created), eq(OwnerScope.USER), isNull(), eq(user), any());
|
||||
@@ -156,6 +159,60 @@ class IntegrationConfigServiceTest {
|
||||
.isEqualTo(HttpStatus.FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---- S3 type policy ----
|
||||
|
||||
@Test
|
||||
void s3PersonalCreateForbiddenForRegularUser() {
|
||||
User user = user(7);
|
||||
when(ownership.isAdmin(user)).thenReturn(false);
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
service.create(
|
||||
request(IntegrationType.S3, OwnerScope.USER, null), user))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.extracting(e -> ((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.FORBIDDEN);
|
||||
}
|
||||
|
||||
@Test
|
||||
void s3PersonalCreateAllowedForAdmin() {
|
||||
when(secretMasker.sanitize(any())).thenReturn(Map.of("bucket", "b"));
|
||||
when(repository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
User admin = user(1);
|
||||
when(ownership.isAdmin(admin)).thenReturn(true);
|
||||
|
||||
IntegrationConfig created =
|
||||
service.create(request(IntegrationType.S3, OwnerScope.USER, null), admin);
|
||||
|
||||
assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.S3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void s3TeamScopeCreateDelegatesLeadershipToOwnership() {
|
||||
// TEAM scope skips the personal-S3 gate; assignOwnership enforces admin/team-owner.
|
||||
when(secretMasker.sanitize(any())).thenReturn(Map.of("bucket", "b"));
|
||||
when(repository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
User leader = user(7);
|
||||
|
||||
IntegrationConfig created =
|
||||
service.create(request(IntegrationType.S3, OwnerScope.TEAM, 3L), leader);
|
||||
|
||||
verify(ownership)
|
||||
.assignOwnership(eq(created), eq(OwnerScope.TEAM), eq(3L), eq(leader), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void mcpPersonalCreateAllowedForRegularUser() {
|
||||
when(secretMasker.sanitize(any())).thenReturn(Map.of("token", "t"));
|
||||
when(repository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
IntegrationConfig created =
|
||||
service.create(request(IntegrationType.MCP, OwnerScope.USER, null), user(7));
|
||||
|
||||
assertThat(created.getIntegrationType()).isEqualTo(IntegrationType.MCP);
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private IntegrationConfig config(long id) {
|
||||
|
||||
+1
-2
@@ -1,4 +1,4 @@
|
||||
package stirling.software.saas.model;
|
||||
package stirling.software.proprietary.model;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -9,7 +9,6 @@ import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.enumeration.TeamRole;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
|
||||
/** Constructor, accessor, equals/hashCode/toString, and role-helper tests for TeamMembership. */
|
||||
+52
@@ -34,6 +34,7 @@ import stirling.software.proprietary.policy.engine.PolicyRunHandle;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.PolicyValidator;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
@@ -62,6 +63,8 @@ class PolicyControllerTest {
|
||||
private stirling.software.proprietary.policy.overview.PolicyOverviewService
|
||||
policyOverviewService;
|
||||
|
||||
@Mock private ProcessedLedger processedLedger;
|
||||
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private JobOwnershipService jobOwnershipService;
|
||||
|
||||
@@ -89,6 +92,7 @@ class PolicyControllerTest {
|
||||
policyManagementAuthority,
|
||||
policyTriggerManager,
|
||||
policyOverviewService,
|
||||
processedLedger,
|
||||
policyTriggers,
|
||||
applicationProperties,
|
||||
tempFileManager,
|
||||
@@ -398,6 +402,7 @@ class PolicyControllerTest {
|
||||
ResponseEntity<Void> response = controller.deletePolicy("a");
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
verify(processedLedger).clearPolicy("a");
|
||||
verify(policyTriggerManager).notifyPoliciesChanged();
|
||||
}
|
||||
|
||||
@@ -431,6 +436,53 @@ class PolicyControllerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("clearProcessedHistory")
|
||||
class ClearProcessedHistory {
|
||||
|
||||
@Test
|
||||
@DisplayName("clears an accessible policy's history")
|
||||
void clears() {
|
||||
applicationProperties.getSecurity().setEnableLogin(false);
|
||||
Policy p = policy("a", 1L);
|
||||
when(policyStore.get("a")).thenReturn(Optional.of(p));
|
||||
when(policyAccessGuard.canAccess(p)).thenReturn(true);
|
||||
|
||||
ResponseEntity<Void> response = controller.clearProcessedHistory("a");
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
verify(processedLedger).clearPolicy("a");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("returns 404 when policy is not accessible")
|
||||
void notAccessible() {
|
||||
applicationProperties.getSecurity().setEnableLogin(false);
|
||||
Policy p = policy("a", 1L);
|
||||
when(policyStore.get("a")).thenReturn(Optional.of(p));
|
||||
when(policyAccessGuard.canAccess(p)).thenReturn(false);
|
||||
|
||||
ResponseEntity<Void> response = controller.clearProcessedHistory("a");
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND);
|
||||
verify(processedLedger, never()).clearPolicy(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("forbidden when login enabled and caller cannot edit")
|
||||
void forbidden() {
|
||||
applicationProperties.getSecurity().setEnableLogin(true);
|
||||
when(policyManagementAuthority.canEditPolicies()).thenReturn(false);
|
||||
|
||||
assertThatThrownBy(() -> controller.clearProcessedHistory("a"))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.FORBIDDEN));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("runStoredPolicy")
|
||||
class RunStoredPolicy {
|
||||
|
||||
+117
-7
@@ -4,15 +4,19 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@@ -24,7 +28,9 @@ import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.ResolveContext;
|
||||
import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
@@ -39,15 +45,15 @@ import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
|
||||
/**
|
||||
* Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs. Verifies
|
||||
* it pulls every source, runs one job per unit of work, feeds each unit's completion hook the run
|
||||
* outcome, and that a generator (no sources) still runs once.
|
||||
* Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs, and the
|
||||
* orchestrator of ledger hygiene (presence stamping + cleanup on complete FULL sweeps).
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PolicyRunnerTest {
|
||||
|
||||
@Mock private PolicyEngine policyEngine;
|
||||
@Mock private InputSource folderSource;
|
||||
@Mock private ProcessedLedger processedLedger;
|
||||
|
||||
private final SourceStore sourceStore = new InProcessSourceStore();
|
||||
private PolicyRunner runner;
|
||||
@@ -59,7 +65,8 @@ class PolicyRunnerTest {
|
||||
policyEngine,
|
||||
List.of(folderSource),
|
||||
sourceStore,
|
||||
new InProcessSourceDocCounter());
|
||||
new InProcessSourceDocCounter(),
|
||||
processedLedger);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -73,6 +80,9 @@ class PolicyRunnerTest {
|
||||
ArgumentCaptor<PolicyInputs> inputs = ArgumentCaptor.forClass(PolicyInputs.class);
|
||||
verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any());
|
||||
assertTrue(inputs.getValue().primary().isEmpty());
|
||||
// Ledger hygiene still runs: rows recorded for a generator policy's folder outputs
|
||||
// are pruned by its own sweeps rather than accumulating until the policy is deleted.
|
||||
verify(processedLedger).deleteUnseen(eq("p1"), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -80,7 +90,7 @@ class PolicyRunnerTest {
|
||||
InputSpec spec = InputSpec.folder("/in");
|
||||
Policy policy = policy(List.of(spec));
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(spec))
|
||||
when(folderSource.resolve(eq(spec), any()))
|
||||
.thenReturn(
|
||||
List.of(
|
||||
ResolvedInput.of(PolicyInputs.of(List.of())),
|
||||
@@ -100,7 +110,7 @@ class PolicyRunnerTest {
|
||||
AtomicBoolean outcome = new AtomicBoolean(false);
|
||||
ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set);
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(spec)).thenReturn(List.of(unit));
|
||||
when(folderSource.resolve(eq(spec), any())).thenReturn(List.of(unit));
|
||||
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
|
||||
when(policyEngine.runPolicy(any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", completion));
|
||||
@@ -121,7 +131,7 @@ class PolicyRunnerTest {
|
||||
AtomicBoolean outcome = new AtomicBoolean(true);
|
||||
ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set);
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(spec)).thenReturn(List.of(unit));
|
||||
when(folderSource.resolve(eq(spec), any())).thenReturn(List.of(unit));
|
||||
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
|
||||
when(policyEngine.runPolicy(any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", completion));
|
||||
@@ -143,6 +153,98 @@ class PolicyRunnerTest {
|
||||
verifyNoInteractions(policyEngine);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFullSweepStampsPresenceAndPrunesUnseenRows() throws Exception {
|
||||
InputSpec spec = InputSpec.folder("/in");
|
||||
Policy policy = policy(List.of(spec));
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.listsExhaustively()).thenReturn(true);
|
||||
when(folderSource.resolve(eq(spec), any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
ResolveContext ctx = invocation.getArgument(1);
|
||||
ctx.reportPresent(List.of("/in/a.pdf", "/in/b.pdf"));
|
||||
return List.of();
|
||||
});
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
// Presence reporting also bulk-prefetches claim state: one lookup for the whole listing.
|
||||
verify(processedLedger).statesFor(eq("p1"), eq(List.of("/in/a.pdf", "/in/b.pdf")));
|
||||
verify(processedLedger).markSeen("p1", Set.of("/in/a.pdf", "/in/b.pdf"));
|
||||
verify(processedLedger).deleteUnseen(eq("p1"), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aLightSweepClaimsButSkipsLedgerHygiene() throws Exception {
|
||||
InputSpec spec = InputSpec.folder("/in");
|
||||
Policy policy = policy(List.of(spec));
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(eq(spec), any()))
|
||||
.thenReturn(List.of(ResolvedInput.of(PolicyInputs.of(List.of()))));
|
||||
when(policyEngine.runPolicy(any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
|
||||
|
||||
runner.run(policy, SweepKind.LIGHT);
|
||||
|
||||
verify(policyEngine).runPolicy(eq(policy), any(), any());
|
||||
verify(processedLedger, never()).markSeen(any(), any());
|
||||
verify(processedLedger, never()).deleteUnseen(any(), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSourceThatFailsToResolveVetoesCleanupButOthersStillRun() throws Exception {
|
||||
InputSpec broken = InputSpec.folder("/broken");
|
||||
InputSpec healthy = InputSpec.folder("/healthy");
|
||||
Policy policy = policy(List.of(broken, healthy));
|
||||
when(folderSource.supports(any())).thenReturn(true);
|
||||
when(folderSource.listsExhaustively()).thenReturn(true);
|
||||
when(folderSource.resolve(eq(broken), any())).thenThrow(new IOException("mount gone"));
|
||||
when(folderSource.resolve(eq(healthy), any()))
|
||||
.thenReturn(List.of(ResolvedInput.of(PolicyInputs.of(List.of()))));
|
||||
when(policyEngine.runPolicy(any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
verify(policyEngine).runPolicy(eq(policy), any(), any()); // healthy source still ran
|
||||
verify(processedLedger, never()).deleteUnseen(any(), anyLong()); // history preserved
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDisabledSourceVetoesCleanup() {
|
||||
InputSpec spec = InputSpec.folder("/in");
|
||||
String pausedId = sourceStore.save(disabledSourceFrom(spec)).id();
|
||||
Policy policy = policyReferencing(List.of(pausedId));
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
verify(processedLedger, never()).deleteUnseen(any(), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aNonExhaustiveSourceVetoesCleanup() throws Exception {
|
||||
InputSpec spec = InputSpec.folder("/in");
|
||||
Policy policy = policy(List.of(spec));
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.listsExhaustively()).thenReturn(false);
|
||||
when(folderSource.resolve(eq(spec), any())).thenReturn(List.of());
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
verify(processedLedger, never()).deleteUnseen(any(), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMissingSourceDoesNotVetoCleanup() {
|
||||
// A deleted source's rows age out precisely because cleanup still runs.
|
||||
Policy policy = policyReferencing(List.of("ghost-source-id"));
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
verify(processedLedger).deleteUnseen(eq("p1"), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runWithSuppliedInputsBypassesSources() {
|
||||
Policy policy = policy(List.of(InputSpec.folder("/in")));
|
||||
@@ -159,6 +261,10 @@ class PolicyRunnerTest {
|
||||
private Policy policy(List<InputSpec> sources) {
|
||||
List<String> sourceIds =
|
||||
sources.stream().map(spec -> sourceStore.save(sourceFrom(spec)).id()).toList();
|
||||
return policyReferencing(sourceIds);
|
||||
}
|
||||
|
||||
private static Policy policyReferencing(List<String> sourceIds) {
|
||||
return new Policy(
|
||||
"p1",
|
||||
"p",
|
||||
@@ -173,4 +279,8 @@ class PolicyRunnerTest {
|
||||
private static Source sourceFrom(InputSpec spec) {
|
||||
return new Source(null, "src", spec.type(), spec.options(), true, "owner", null);
|
||||
}
|
||||
|
||||
private static Source disabledSourceFrom(InputSpec spec) {
|
||||
return new Source(null, "src", spec.type(), spec.options(), false, "owner", null);
|
||||
}
|
||||
}
|
||||
|
||||
+291
-35
@@ -1,17 +1,23 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -24,18 +30,26 @@ import org.springframework.core.env.StandardEnvironment;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
|
||||
/** Tests for {@link FolderInputSource}: consume (claim + route) and snapshot (read-only) modes. */
|
||||
/**
|
||||
* Tests for {@link FolderInputSource}: consume mode tracks files in place through the ledger,
|
||||
* snapshot stays stateless, and discovery skips hidden entries and honours the recursive option.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class FolderInputSourceTest {
|
||||
|
||||
private static final String POLICY = "p1";
|
||||
|
||||
@Mock private FileReadinessChecker readinessChecker;
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private FolderInputSource source;
|
||||
private InProcessProcessedLedger ledger;
|
||||
private RecordingContext ctx;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
@@ -45,73 +59,279 @@ class FolderInputSourceTest {
|
||||
new FolderAccessGuard(
|
||||
properties, new StandardEnvironment(), new InProcessSourceStore());
|
||||
source = new FolderInputSource(readinessChecker, guard);
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
// Lenient: the missing-dir / nonexistent-dir cases return before any readiness check.
|
||||
lenient().when(readinessChecker.isReady(any())).thenReturn(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumeClaimsFilesAndRoutesToDoneOnSuccess() throws IOException {
|
||||
void consumeRemovesTheFileOnceProcessed() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Files.writeString(inputDir.resolve("doc.pdf"), "data");
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
|
||||
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()));
|
||||
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()), ctx);
|
||||
|
||||
assertEquals(1, work.size());
|
||||
assertEquals(1, work.get(0).inputs().primary().size());
|
||||
// Claimed out of the input dir.
|
||||
assertFalse(Files.exists(inputDir.resolve("doc.pdf")));
|
||||
assertTrue(
|
||||
Files.exists(
|
||||
inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf")));
|
||||
// In flight: still on disk, but a second sweep does not pick it up again.
|
||||
assertTrue(Files.exists(file));
|
||||
assertTrue(Files.notExists(inputDir.resolve(".stirling")));
|
||||
assertTrue(source.resolve(InputSpec.folder(inputDir.toString()), ctx).isEmpty());
|
||||
|
||||
work.get(0).onComplete().accept(true);
|
||||
assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("done").resolve("doc.pdf")));
|
||||
assertFalse(
|
||||
Files.exists(
|
||||
inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf")));
|
||||
assertTrue(Files.notExists(file));
|
||||
assertTrue(source.resolve(InputSpec.folder(inputDir.toString()), ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumeRoutesToErrorOnFailure() throws IOException {
|
||||
void aFileReplacedMidRunSurvivesTheDeleteAndRunsAgain() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Files.writeString(inputDir.resolve("doc.pdf"), "data");
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
|
||||
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()));
|
||||
work.get(0).onComplete().accept(false);
|
||||
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()), ctx);
|
||||
// The user saves a new version while the run is executing.
|
||||
Files.writeString(file, "new data, different size");
|
||||
work.get(0).onComplete().accept(true);
|
||||
|
||||
assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("error").resolve("doc.pdf")));
|
||||
// The delete is version-guarded: the replacement is not the file that ran, so it stays
|
||||
// and is claimed as fresh work instead of being marked processed.
|
||||
assertTrue(Files.exists(file));
|
||||
assertEquals(1, source.resolve(InputSpec.folder(inputDir.toString()), ctx).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void snapshotReadsWithoutClaiming() throws IOException {
|
||||
void aSharedFileIsRemovedOnlyOnceEveryPolicyHasProcessedIt() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
InputSpec spec = InputSpec.folder(inputDir.toString());
|
||||
RecordingContext other = new RecordingContext("p2");
|
||||
|
||||
List<ResolvedInput> mine = source.resolve(spec, ctx);
|
||||
List<ResolvedInput> theirs = source.resolve(spec, other);
|
||||
assertEquals(1, mine.size());
|
||||
assertEquals(1, theirs.size());
|
||||
|
||||
mine.get(0).onComplete().accept(true);
|
||||
// The other policy's claim is still in flight, so the first finisher must not delete.
|
||||
assertTrue(Files.exists(file));
|
||||
|
||||
theirs.get(0).onComplete().accept(true);
|
||||
assertTrue(Files.notExists(file));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSharedFileStaysParkedWhenAnyPolicyFailsOnIt() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
InputSpec spec = InputSpec.folder(inputDir.toString());
|
||||
RecordingContext other = new RecordingContext("p2");
|
||||
|
||||
List<ResolvedInput> mine = source.resolve(spec, ctx);
|
||||
List<ResolvedInput> theirs = source.resolve(spec, other);
|
||||
|
||||
theirs.get(0).onComplete().accept(false);
|
||||
mine.get(0).onComplete().accept(true);
|
||||
|
||||
// The failure parks the file for everyone (retried when it changes), regardless of
|
||||
// which policy settled last.
|
||||
assertTrue(Files.exists(file));
|
||||
assertTrue(source.resolve(spec, ctx).isEmpty());
|
||||
assertTrue(source.resolve(spec, other).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReDroppedFileIsProcessedAgain() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
|
||||
source.resolve(InputSpec.folder(inputDir.toString()), ctx).get(0).onComplete().accept(true);
|
||||
Files.writeString(file, "data again");
|
||||
|
||||
assertEquals(1, source.resolve(InputSpec.folder(inputDir.toString()), ctx).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedFileStaysInPlaceAndIsNotRetriedUntilItChanges() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
|
||||
source.resolve(InputSpec.folder(inputDir.toString()), ctx)
|
||||
.get(0)
|
||||
.onComplete()
|
||||
.accept(false);
|
||||
|
||||
assertTrue(Files.exists(file));
|
||||
assertTrue(source.resolve(InputSpec.folder(inputDir.toString()), ctx).isEmpty());
|
||||
|
||||
Files.setLastModifiedTime(file, FileTime.from(Instant.now().plusSeconds(60)));
|
||||
assertEquals(1, source.resolve(InputSpec.folder(inputDir.toString()), ctx).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void statModeRetriesAFailureOnATouchButHashModeDoesNot() throws IOException {
|
||||
Path statDir = Files.createDirectories(tempDir.resolve("stat"));
|
||||
Path hashDir = Files.createDirectories(tempDir.resolve("hash"));
|
||||
Path statFile = statDir.resolve("doc.pdf");
|
||||
Path hashFile = hashDir.resolve("doc.pdf");
|
||||
Files.writeString(statFile, "data");
|
||||
Files.writeString(hashFile, "data");
|
||||
InputSpec statSpec = InputSpec.folder(statDir.toString());
|
||||
InputSpec hashSpec =
|
||||
new InputSpec(
|
||||
"folder", Map.of("directory", hashDir.toString(), "identity", "hash"));
|
||||
|
||||
source.resolve(statSpec, ctx).get(0).onComplete().accept(false);
|
||||
source.resolve(hashSpec, ctx).get(0).onComplete().accept(false);
|
||||
|
||||
FileTime touched = FileTime.from(Instant.now().plusSeconds(60));
|
||||
Files.setLastModifiedTime(statFile, touched);
|
||||
Files.setLastModifiedTime(hashFile, touched);
|
||||
|
||||
// Same content, new mtime: stat mode calls that a new version and retries; hash mode
|
||||
// verifies the content is unchanged and keeps the failure parked.
|
||||
assertEquals(1, source.resolve(statSpec, ctx).size());
|
||||
assertTrue(source.resolve(hashSpec, ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void hashModeRetriesAFailureOnARealContentChange() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path file = inputDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
InputSpec spec =
|
||||
new InputSpec(
|
||||
"folder", Map.of("directory", inputDir.toString(), "identity", "hash"));
|
||||
|
||||
source.resolve(spec, ctx).get(0).onComplete().accept(false);
|
||||
Files.writeString(file, "data v2 - longer");
|
||||
|
||||
assertEquals(1, source.resolve(spec, ctx).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void snapshotReadsStatelesslyEverySweep() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Files.writeString(inputDir.resolve("doc.pdf"), "data");
|
||||
InputSpec spec =
|
||||
new InputSpec(
|
||||
"folder", Map.of("directory", inputDir.toString(), "mode", "snapshot"));
|
||||
|
||||
List<ResolvedInput> work =
|
||||
source.resolve(
|
||||
new InputSpec(
|
||||
"folder",
|
||||
Map.of("directory", inputDir.toString(), "mode", "snapshot")));
|
||||
List<ResolvedInput> first = source.resolve(spec, ctx);
|
||||
first.get(0).onComplete().accept(true);
|
||||
List<ResolvedInput> second = source.resolve(spec, ctx);
|
||||
|
||||
assertEquals(1, first.size());
|
||||
assertEquals(1, second.size()); // no ledger involvement: every run sees the full set
|
||||
assertTrue(ctx.present.isEmpty());
|
||||
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void hiddenFilesAndTheLegacyWorkDirAreIgnored() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Files.writeString(inputDir.resolve("doc.pdf"), "data");
|
||||
Files.writeString(inputDir.resolve(".hidden.pdf"), "secret");
|
||||
Path legacy = Files.createDirectories(inputDir.resolve(".stirling").resolve("done"));
|
||||
Files.writeString(legacy.resolve("old.pdf"), "processed long ago");
|
||||
|
||||
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()), ctx);
|
||||
|
||||
assertEquals(1, work.size());
|
||||
// Not moved, and completing the run is a no-op.
|
||||
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
|
||||
work.get(0).onComplete().accept(true);
|
||||
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
|
||||
assertEquals(1, ctx.present.size());
|
||||
assertTrue(ctx.present.get(0).endsWith("doc.pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recursiveDiscoversSubdirectoriesButNotHiddenOnes() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Files.writeString(inputDir.resolve("top.pdf"), "a");
|
||||
Path sub = Files.createDirectories(inputDir.resolve("sub"));
|
||||
Files.writeString(sub.resolve("nested.pdf"), "b");
|
||||
Path hiddenDir = Files.createDirectories(inputDir.resolve(".stirling"));
|
||||
Files.writeString(hiddenDir.resolve("skipped.pdf"), "c");
|
||||
// Sink staging inside a watched subdirectory is pruned at any depth.
|
||||
Path nestedStaging = Files.createDirectories(sub.resolve(".stirling").resolve("tmp"));
|
||||
Files.writeString(nestedStaging.resolve("half-delivered"), "d");
|
||||
|
||||
InputSpec flat = InputSpec.folder(inputDir.toString());
|
||||
InputSpec recursive =
|
||||
new InputSpec(
|
||||
"folder", Map.of("directory", inputDir.toString(), "recursive", "true"));
|
||||
|
||||
assertEquals(1, source.resolve(flat, ctx).size());
|
||||
assertEquals(1, source.resolve(recursive, ctx).size()); // top.pdf already claimed above
|
||||
assertTrue(ctx.present.stream().anyMatch(identity -> identity.endsWith("nested.pdf")));
|
||||
assertTrue(ctx.present.stream().noneMatch(identity -> identity.endsWith("skipped.pdf")));
|
||||
assertTrue(ctx.present.stream().noneMatch(identity -> identity.endsWith("half-delivered")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unreadyFilesAreReportedPresentButNotClaimed() throws IOException {
|
||||
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path file = inputDir.resolve("mid-write.pdf");
|
||||
Files.writeString(file, "partial");
|
||||
when(readinessChecker.isReady(file)).thenReturn(false);
|
||||
|
||||
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()), ctx);
|
||||
|
||||
assertTrue(work.isEmpty());
|
||||
// Reported present so a full sweep does not prune its row while it settles on disk.
|
||||
assertEquals(1, ctx.present.size());
|
||||
assertTrue(ctx.present.get(0).endsWith("mid-write.pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nestedSourcesShareThePolicysLedgerAndDoNotDoubleClaim() throws IOException {
|
||||
Path parent = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path child = Files.createDirectories(parent.resolve("sub"));
|
||||
Files.writeString(child.resolve("doc.pdf"), "data");
|
||||
InputSpec parentRecursive =
|
||||
new InputSpec(
|
||||
"folder", Map.of("directory", parent.toString(), "recursive", "true"));
|
||||
InputSpec childFlat = InputSpec.folder(child.toString());
|
||||
|
||||
// Same sweep, same policy context: whichever source resolves first wins the file.
|
||||
assertEquals(1, source.resolve(parentRecursive, ctx).size());
|
||||
assertTrue(source.resolve(childFlat, ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingDirectoryOptionFails() {
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> source.resolve(new InputSpec("folder", Map.of())));
|
||||
() -> source.resolve(new InputSpec("folder", Map.of()), ctx));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonexistentDirectoryYieldsNoWork() throws IOException {
|
||||
List<ResolvedInput> work =
|
||||
source.resolve(InputSpec.folder(tempDir.resolve("nope").toString()));
|
||||
assertTrue(work.isEmpty());
|
||||
void anUnknownIdentityModeIsRejected() {
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
source.validate(
|
||||
new InputSpec(
|
||||
"folder",
|
||||
Map.of(
|
||||
"directory",
|
||||
tempDir.toString(),
|
||||
"identity",
|
||||
"guesswork"))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonexistentDirectoryFailsResolveSoTheSweepVetoesCleanup() {
|
||||
// An unreachable directory (e.g. unmounted drive) must surface as a failed listing, not
|
||||
// an empty one: the runner vetoes presence cleanup on failure, keeping the history that
|
||||
// an empty listing would wipe.
|
||||
assertThrows(
|
||||
NoSuchFileException.class,
|
||||
() -> source.resolve(InputSpec.folder(tempDir.resolve("nope").toString()), ctx));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -126,7 +346,7 @@ class FolderInputSourceTest {
|
||||
Path outside = tempDir.resolveSibling("not-allowed");
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> source.resolve(InputSpec.folder(outside.toString())));
|
||||
() -> source.resolve(InputSpec.folder(outside.toString()), ctx));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> source.validate(InputSpec.folder(outside.toString())));
|
||||
@@ -137,4 +357,40 @@ class FolderInputSourceTest {
|
||||
Path inputDir = tempDir.resolve("in");
|
||||
assertEquals(List.of(inputDir), source.watchTargets(InputSpec.folder(inputDir.toString())));
|
||||
}
|
||||
|
||||
/** Policy-scoped context backed by the in-process ledger, recording presence reports. */
|
||||
private class RecordingContext implements ResolveContext {
|
||||
|
||||
private final String policyId;
|
||||
private final List<String> present = new ArrayList<>();
|
||||
|
||||
private RecordingContext() {
|
||||
this(POLICY);
|
||||
}
|
||||
|
||||
private RecordingContext(String policyId) {
|
||||
this.policyId = policyId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean claim(String identity, String gate, Supplier<String> contentHash) {
|
||||
return ledger.claim(policyId, identity, gate, contentHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(
|
||||
String identity, String finalGate, String finalContentHash, boolean success) {
|
||||
ledger.settle(policyId, identity, finalGate, finalContentHash, success);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allSettledDone(String identity) {
|
||||
return ledger.allSettledDone(identity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reportPresent(Collection<String> identities) {
|
||||
present.addAll(identities);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.time.Instant;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
/**
|
||||
* Tests for {@link FolderIdentities}: identity derivation must agree for the same file, even
|
||||
* through a symlinked alias of the directory.
|
||||
*/
|
||||
class FolderIdentitiesTest {
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
@Test
|
||||
void identityAgreesAcrossASymlinkedAliasOfTheDirectory() throws IOException {
|
||||
Path real = Files.createDirectories(tempDir.resolve("real"));
|
||||
Path alias = Files.createSymbolicLink(tempDir.resolve("alias"), real);
|
||||
Files.writeString(real.resolve("doc.pdf"), "data");
|
||||
|
||||
String viaReal =
|
||||
FolderIdentities.identity(
|
||||
FolderIdentities.canonicalDir(real), real, real.resolve("doc.pdf"));
|
||||
String viaAlias =
|
||||
FolderIdentities.identity(
|
||||
FolderIdentities.canonicalDir(alias), alias, alias.resolve("doc.pdf"));
|
||||
|
||||
assertEquals(viaReal, viaAlias);
|
||||
}
|
||||
|
||||
@Test
|
||||
void identityOfANestedFileKeepsItsRelativePath() throws IOException {
|
||||
Path dir = Files.createDirectories(tempDir.resolve("in"));
|
||||
Path nested = Files.createDirectories(dir.resolve("sub")).resolve("doc.pdf");
|
||||
Files.writeString(nested, "data");
|
||||
|
||||
String identity =
|
||||
FolderIdentities.identity(FolderIdentities.canonicalDir(dir), dir, nested);
|
||||
|
||||
assertTrue(identity.endsWith("sub" + java.io.File.separator + "doc.pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void theGateTracksSizeAndMtime() throws IOException {
|
||||
Path file = tempDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
String before = FolderIdentities.statGate(file);
|
||||
|
||||
Files.setLastModifiedTime(file, FileTime.from(Instant.now().plusSeconds(60)));
|
||||
|
||||
assertNotEquals(before, FolderIdentities.statGate(file));
|
||||
}
|
||||
|
||||
@Test
|
||||
void theContentHashIgnoresMtimeButTracksContent() throws IOException {
|
||||
Path file = tempDir.resolve("doc.pdf");
|
||||
Files.writeString(file, "data");
|
||||
String before = FolderIdentities.contentHash(file);
|
||||
|
||||
Files.setLastModifiedTime(file, FileTime.from(Instant.now().plusSeconds(60)));
|
||||
assertEquals(before, FolderIdentities.contentHash(file));
|
||||
|
||||
Files.writeString(file, "different");
|
||||
assertNotEquals(before, FolderIdentities.contentHash(file));
|
||||
}
|
||||
|
||||
@Test
|
||||
void identityHashIsAStableFixedWidthKey() {
|
||||
String hash = IdentityHasher.identityHash("/in/doc.pdf");
|
||||
|
||||
assertEquals(64, hash.length());
|
||||
assertEquals(hash, IdentityHasher.identityHash("/in/doc.pdf"));
|
||||
assertNotEquals(hash, IdentityHasher.identityHash("/in/other.pdf"));
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/** {@link InProcessProcessedLedger} against the shared {@link ProcessedLedger} contract. */
|
||||
class InProcessProcessedLedgerTest extends ProcessedLedgerContractTest {
|
||||
|
||||
@Override
|
||||
ProcessedLedger newLedger(Supplier<Long> nowMillis) {
|
||||
return new InProcessProcessedLedger(nowMillis);
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurationPackage;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
|
||||
/**
|
||||
* {@link JpaProcessedLedger} against the shared contract on a real (H2) database. The inherited
|
||||
* tests run outside {@code @DataJpaTest}'s per-test transaction (the transaction attribute resolves
|
||||
* against the declaring class, the plain contract base), so every ledger call commits in its own
|
||||
* transaction as at runtime; state is wiped explicitly instead of relying on rollback.
|
||||
*/
|
||||
@DataJpaTest
|
||||
class JpaProcessedLedgerDbTest extends ProcessedLedgerContractTest {
|
||||
|
||||
@Autowired private ProcessedFileRepository repository;
|
||||
|
||||
@AfterEach
|
||||
void wipeLedger() {
|
||||
// deleteAll() skips entities whose isNew() is hardcoded true, so use the bulk form.
|
||||
repository.deleteAllInBatch();
|
||||
}
|
||||
|
||||
@Override
|
||||
ProcessedLedger newLedger(Supplier<Long> nowMillis) {
|
||||
return new JpaProcessedLedger(repository, nowMillis);
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
@AutoConfigurationPackage
|
||||
static class TestApp {}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** The {@link ProcessedLedger} contract, run against every implementation so they cannot drift. */
|
||||
abstract class ProcessedLedgerContractTest {
|
||||
|
||||
static final String POLICY = "p1";
|
||||
static final String OTHER_POLICY = "p2";
|
||||
static final String FILE = "/in/doc.pdf";
|
||||
static final String GATE = "100:1111";
|
||||
static final String NEW_GATE = "100:2222";
|
||||
static final String HASH = "hash-aaa";
|
||||
static final String NEW_HASH = "hash-bbb";
|
||||
|
||||
final AtomicLong clock = new AtomicLong(1_000_000L);
|
||||
|
||||
ProcessedLedger ledger;
|
||||
|
||||
abstract ProcessedLedger newLedger(Supplier<Long> nowMillis);
|
||||
|
||||
@BeforeEach
|
||||
void createLedger() {
|
||||
ledger = newLedger(clock::get);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFileIsClaimedOnceAndSkippedWhileInFlight() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, null)); // new version waits for settle
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSettledFileIsSkippedAtTheSameGate() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMovedGateIsReclaimedInGateOnlyMode() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void settlingAtTheOutputsVersionStopsAnInPlaceOverwriteLooping() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
// The run overwrote the input; settle re-reads and lands on the produced version.
|
||||
ledger.settle(POLICY, FILE, NEW_GATE, null, true);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, null)); // own output: skip
|
||||
assertTrue(ledger.claim(POLICY, FILE, "100:3333", null)); // later user edit: reprocess
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedFileIsNotRetriedUntilItChanges() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.settle(POLICY, FILE, GATE, null, false);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTouchedButUnchangedFileRefreshesTheGateInsteadOfReprocessing() {
|
||||
ledger.claim(POLICY, FILE, GATE, hash(HASH));
|
||||
ledger.settle(POLICY, FILE, GATE, HASH, true);
|
||||
|
||||
// Same content under a new gate (touch / identical re-copy): verified, not reprocessed.
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH)));
|
||||
|
||||
// The gate was refreshed, so the next sweep takes the cheap path: no content read at all.
|
||||
CountingSupplier counting = new CountingSupplier(HASH);
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, counting));
|
||||
assertEquals(0, counting.invocations.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void theVerificationTierIsNotConsultedWhileTheGateMatches() {
|
||||
ledger.claim(POLICY, FILE, GATE, hash(HASH));
|
||||
ledger.settle(POLICY, FILE, GATE, HASH, true);
|
||||
|
||||
CountingSupplier counting = new CountingSupplier(HASH);
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, counting));
|
||||
assertEquals(0, counting.invocations.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRealContentChangeUnderANewGateIsReprocessed() {
|
||||
ledger.claim(POLICY, FILE, GATE, hash(HASH));
|
||||
ledger.settle(POLICY, FILE, GATE, HASH, true);
|
||||
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, hash(NEW_HASH)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aGateOnlySettledRowCannotBeContentVerifiedSoItReprocesses() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
|
||||
// The row stored no hash, so "same content" is unprovable: reprocess on gate change.
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedFileStaysParkedThroughATouch() {
|
||||
ledger.claim(POLICY, FILE, GATE, hash(HASH));
|
||||
ledger.settle(POLICY, FILE, GATE, HASH, false);
|
||||
|
||||
// A touch must not resurrect an ERROR row; only a real content change does.
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH)));
|
||||
assertTrue(ledger.claim(POLICY, FILE, "100:3333", hash(NEW_HASH)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void interruptedRunsAreRetriedABoundedNumberOfTimes() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // attempt 1 dies with the JVM
|
||||
ledger.recoverInterrupted();
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // attempt 2
|
||||
ledger.recoverInterrupted();
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // attempt 3, the last
|
||||
ledger.recoverInterrupted();
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null)); // parked: no crash-loop
|
||||
}
|
||||
|
||||
@Test
|
||||
void aNewGateResetsTheInterruptRetryBudgetInGateOnlyMode() {
|
||||
for (int attempt = 0; attempt < ProcessedLedger.MAX_ATTEMPTS; attempt++) {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.recoverInterrupted();
|
||||
}
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null));
|
||||
ledger.recoverInterrupted();
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); // fresh budget at the new version
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTouchDoesNotResetTheInterruptRetryBudgetWhenContentIsVerified() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, hash(HASH))); // attempt 1
|
||||
ledger.recoverInterrupted();
|
||||
// Same content, moved gate: still the interrupted work, still bounded.
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH))); // attempt 2
|
||||
ledger.recoverInterrupted();
|
||||
assertTrue(ledger.claim(POLICY, FILE, "100:3333", hash(HASH))); // attempt 3
|
||||
ledger.recoverInterrupted();
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, "100:4444", hash(HASH))); // parked
|
||||
assertTrue(ledger.claim(POLICY, FILE, "100:5555", hash(NEW_HASH))); // real change: fresh
|
||||
}
|
||||
|
||||
@Test
|
||||
void recoveryOnlyTouchesInFlightRows() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
ledger.recoverInterrupted();
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null)); // still DONE, not retried
|
||||
}
|
||||
|
||||
@Test
|
||||
void anOutputIsSkippedByItsProducerButSeenByOtherPolicies() {
|
||||
ledger.recordOutput(POLICY, FILE, GATE, HASH);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null)); // producer skips its own output
|
||||
assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null)); // chaining still works
|
||||
}
|
||||
|
||||
@Test
|
||||
void anOutputIsSkippedByAHashVerifyingProducerEvenIfTheGateMoved() {
|
||||
ledger.recordOutput(POLICY, FILE, GATE, HASH);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, hash(HASH)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void policiesTrackTheSameFileIndependently() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void statesForSnapshotsOnlyExistingRows() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
|
||||
Map<String, ClaimState> states = ledger.statesFor(POLICY, List.of(FILE, "/in/other.pdf"));
|
||||
|
||||
assertEquals(1, states.size());
|
||||
assertEquals(ProcessedFileStatus.PROCESSING, states.get(FILE).status());
|
||||
assertEquals(GATE, states.get(FILE).gate());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aStaleAbsentSnapshotLosesTheClaimRaceInsteadOfDoubleClaiming() {
|
||||
ClaimState absent = ledger.statesFor(POLICY, List.of(FILE)).get(FILE); // no row yet
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null)); // another sweep wins meanwhile
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null, absent));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aStaleSettledSnapshotCannotReclaimAnInFlightRow() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
ClaimState settled = ledger.statesFor(POLICY, List.of(FILE)).get(FILE);
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); // a fresh sweep reclaims first
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, "100:3333", null, settled));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aForgottenOutputIsClaimableAtAnyVersion() {
|
||||
ledger.recordOutput(POLICY, FILE, GATE, HASH);
|
||||
ledger.forgetOutput(POLICY, FILE, GATE);
|
||||
|
||||
// Even a byte-identical file at that identity is fresh work: the record is gone.
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, hash(HASH)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void forgetOutputLeavesARowReclaimedInTheMeantime() {
|
||||
ledger.recordOutput(POLICY, FILE, GATE, HASH);
|
||||
assertTrue(ledger.claim(POLICY, FILE, NEW_GATE, null)); // a real claim took the row over
|
||||
|
||||
ledger.forgetOutput(POLICY, FILE, GATE);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, NEW_GATE, null)); // still in flight, not deleted
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletionConsensusNeedsEveryClaimantSettledDone() {
|
||||
assertTrue(ledger.allSettledDone(FILE)); // vacuous: no rows yet
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null));
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
assertFalse(ledger.allSettledDone(FILE)); // the other claim is still in flight
|
||||
ledger.settle(OTHER_POLICY, FILE, GATE, null, true);
|
||||
assertTrue(ledger.allSettledDone(FILE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedClaimVetoesDeletionConsensus() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertTrue(ledger.claim(OTHER_POLICY, FILE, GATE, null));
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
ledger.settle(OTHER_POLICY, FILE, GATE, null, false);
|
||||
assertFalse(ledger.allSettledDone(FILE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void anInterruptedClaimVetoesDeletionConsensus() {
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
ledger.recoverInterrupted();
|
||||
assertFalse(ledger.allSettledDone(FILE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void settleRecreatesARowRemovedMidRun() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.clearPolicy(POLICY); // e.g. a clear-history while the run is in flight
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void presenceCleanupRemovesOnlyUnseenSettledRows() {
|
||||
String inFlight = "/in/in-flight.pdf";
|
||||
String stillPresent = "/in/still-present.pdf";
|
||||
String deleted = "/in/deleted.pdf";
|
||||
ledger.claim(POLICY, inFlight, GATE, null);
|
||||
ledger.recordOutput(POLICY, stillPresent, GATE, HASH);
|
||||
ledger.recordOutput(POLICY, deleted, GATE, HASH);
|
||||
|
||||
clock.addAndGet(10_000);
|
||||
long sweepStart = clock.get();
|
||||
ledger.markSeen(POLICY, List.of(inFlight, stillPresent)); // deleted.pdf is gone from disk
|
||||
assertEquals(1, ledger.deleteUnseen(POLICY, sweepStart));
|
||||
|
||||
assertFalse(ledger.claim(POLICY, inFlight, GATE, null)); // in-flight row survived
|
||||
assertFalse(ledger.claim(POLICY, stillPresent, GATE, null)); // stamped row survived
|
||||
assertTrue(ledger.claim(POLICY, deleted, GATE, null)); // forgotten: a re-drop reprocesses
|
||||
}
|
||||
|
||||
@Test
|
||||
void presenceCleanupNeverRemovesInFlightRowsEvenUnstamped() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
clock.addAndGet(10_000);
|
||||
|
||||
assertEquals(0, ledger.deleteUnseen(POLICY, clock.get()));
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rowsWrittenDuringTheSweepSurviveItsCleanup() {
|
||||
long sweepStart = clock.get();
|
||||
// Recorded after the sweep's cutoff: unseen by it, but newer, so it must survive.
|
||||
clock.addAndGet(5);
|
||||
ledger.recordOutput(POLICY, FILE, GATE, HASH);
|
||||
|
||||
assertEquals(0, ledger.deleteUnseen(POLICY, sweepStart));
|
||||
assertFalse(ledger.claim(POLICY, FILE, GATE, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void markSeenOnUnknownIdentitiesIsANoOp() {
|
||||
ledger.markSeen(POLICY, List.of("/never/claimed.pdf"));
|
||||
assertEquals(0, ledger.deleteUnseen(POLICY, clock.get()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void clearPolicyForgetsOnlyThatPolicy() {
|
||||
ledger.claim(POLICY, FILE, GATE, null);
|
||||
ledger.settle(POLICY, FILE, GATE, null, true);
|
||||
ledger.claim(OTHER_POLICY, FILE, GATE, null);
|
||||
ledger.settle(OTHER_POLICY, FILE, GATE, null, true);
|
||||
|
||||
ledger.clearPolicy(POLICY);
|
||||
|
||||
assertTrue(ledger.claim(POLICY, FILE, GATE, null));
|
||||
assertFalse(ledger.claim(OTHER_POLICY, FILE, GATE, null));
|
||||
}
|
||||
|
||||
static Supplier<String> hash(String value) {
|
||||
return () -> value;
|
||||
}
|
||||
|
||||
static final class CountingSupplier implements Supplier<String> {
|
||||
final AtomicInteger invocations = new AtomicInteger();
|
||||
private final String value;
|
||||
|
||||
CountingSupplier(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String get() {
|
||||
invocations.incrementAndGet();
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
+103
-8
@@ -10,6 +10,7 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -21,24 +22,35 @@ import org.springframework.core.io.Resource;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.ledger.FolderIdentities;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
|
||||
/** Tests for {@link FolderOutputSink}: outputs are written to the configured directory on disk. */
|
||||
/**
|
||||
* Tests for {@link FolderOutputSink}: outputs are staged hidden, recorded in the ledger, then
|
||||
* atomically renamed into the configured directory.
|
||||
*/
|
||||
class FolderOutputSinkTest {
|
||||
|
||||
private static final OutputDelivery AD_HOC = new OutputDelivery("run-1", null);
|
||||
private static final OutputDelivery POLICY_RUN = new OutputDelivery("run-1", "p1");
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private FolderOutputSink sink;
|
||||
private InProcessProcessedLedger ledger;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString()));
|
||||
ledger = new InProcessProcessedLedger();
|
||||
sink =
|
||||
new FolderOutputSink(
|
||||
new FolderAccessGuard(
|
||||
properties, new StandardEnvironment(), new InProcessSourceStore()));
|
||||
properties, new StandardEnvironment(), new InProcessSourceStore()),
|
||||
ledger);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -46,13 +58,80 @@ class FolderOutputSinkTest {
|
||||
Path out = tempDir.resolve("out");
|
||||
List<Resource> outputs = List.of(named("a.pdf", "aaa"), named("b.pdf", "bb"));
|
||||
|
||||
List<ResultFile> results =
|
||||
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
|
||||
List<ResultFile> results = sink.deliver(AD_HOC, outputs, OutputSpec.folder(out.toString()));
|
||||
|
||||
assertEquals(2, results.size());
|
||||
assertTrue(Files.exists(out.resolve("a.pdf")));
|
||||
assertEquals("aaa", Files.readString(out.resolve("a.pdf")));
|
||||
assertEquals("bb", Files.readString(out.resolve("b.pdf")));
|
||||
// Nothing left behind in the staging dir.
|
||||
try (Stream<Path> staged = Files.list(out.resolve(".stirling").resolve("tmp"))) {
|
||||
assertEquals(0, staged.count());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordsThePolicysOutputsSoOnlyOtherPoliciesReprocessThem() throws IOException {
|
||||
Path out = tempDir.resolve("out");
|
||||
|
||||
sink.deliver(POLICY_RUN, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString()));
|
||||
|
||||
Path delivered = FolderIdentities.canonicalDir(out).resolve("a.pdf");
|
||||
String gate = FolderIdentities.statGate(delivered);
|
||||
assertFalse(ledger.claim("p1", delivered.toString(), gate, null)); // producer skips it
|
||||
assertTrue(ledger.claim("p2", delivered.toString(), gate, null)); // chaining still works
|
||||
}
|
||||
|
||||
@Test
|
||||
void aHashVerifyingProducerSkipsItsOwnOutputEvenIfTheGateMoved() throws IOException {
|
||||
Path out = tempDir.resolve("out");
|
||||
|
||||
sink.deliver(POLICY_RUN, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString()));
|
||||
|
||||
// A hash-verifying reader matches on content even when the stat moved.
|
||||
Path delivered = FolderIdentities.canonicalDir(out).resolve("a.pdf");
|
||||
assertFalse(
|
||||
ledger.claim(
|
||||
"p1",
|
||||
delivered.toString(),
|
||||
"999:12345",
|
||||
() -> {
|
||||
try {
|
||||
return FolderIdentities.contentHash(delivered);
|
||||
} catch (IOException e) {
|
||||
throw new java.io.UncheckedIOException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordsAnOutputBeforeItBecomesVisible() throws IOException {
|
||||
Path out = tempDir.resolve("out");
|
||||
VisibilityAssertingLedger orderedLedger = new VisibilityAssertingLedger();
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString()));
|
||||
FolderOutputSink orderedSink =
|
||||
new FolderOutputSink(
|
||||
new FolderAccessGuard(
|
||||
properties, new StandardEnvironment(), new InProcessSourceStore()),
|
||||
orderedLedger);
|
||||
|
||||
orderedSink.deliver(
|
||||
POLICY_RUN, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString()));
|
||||
|
||||
assertTrue(orderedLedger.recorded);
|
||||
assertTrue(Files.exists(out.resolve("a.pdf")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void adHocDeliveriesRecordNothing() throws IOException {
|
||||
Path out = tempDir.resolve("out");
|
||||
|
||||
sink.deliver(AD_HOC, List.of(named("a.pdf", "aaa")), OutputSpec.folder(out.toString()));
|
||||
|
||||
Path delivered = FolderIdentities.canonicalDir(out).resolve("a.pdf");
|
||||
// No row was recorded, so any policy (including a hypothetical producer) may claim it.
|
||||
assertTrue(ledger.claim("p1", delivered.toString(), "any-gate", null));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -60,7 +139,7 @@ class FolderOutputSinkTest {
|
||||
Path out = tempDir.resolve("out");
|
||||
List<Resource> outputs = List.of(named("a.pdf", "first"), named("a.pdf", "second"));
|
||||
|
||||
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
|
||||
sink.deliver(AD_HOC, outputs, OutputSpec.folder(out.toString()));
|
||||
|
||||
assertTrue(Files.exists(out.resolve("a.pdf")));
|
||||
assertTrue(Files.exists(out.resolve("a (1).pdf")));
|
||||
@@ -72,7 +151,7 @@ class FolderOutputSinkTest {
|
||||
assertThrows(IllegalArgumentException.class, () -> sink.validate(noDir));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> sink.deliver("run-1", List.of(named("a.pdf", "x")), noDir));
|
||||
() -> sink.deliver(AD_HOC, List.of(named("a.pdf", "x")), noDir));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -81,7 +160,7 @@ class FolderOutputSinkTest {
|
||||
assertThrows(IllegalArgumentException.class, () -> sink.validate(outside));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> sink.deliver("run-1", List.of(named("a.pdf", "x")), outside));
|
||||
() -> sink.deliver(AD_HOC, List.of(named("a.pdf", "x")), outside));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -90,7 +169,7 @@ class FolderOutputSinkTest {
|
||||
List<Resource> outputs =
|
||||
List.of(named("../escape.pdf", "x"), named("nested/deep.pdf", "y"));
|
||||
|
||||
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
|
||||
sink.deliver(AD_HOC, outputs, OutputSpec.folder(out.toString()));
|
||||
|
||||
// Each name is reduced to its bare form inside the target dir; nothing escapes.
|
||||
assertTrue(Files.exists(out.resolve("escape.pdf")));
|
||||
@@ -106,4 +185,20 @@ class FolderOutputSinkTest {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Fails the delivery if an output is visible at its final path before being recorded. */
|
||||
private static class VisibilityAssertingLedger extends InProcessProcessedLedger {
|
||||
|
||||
private boolean recorded;
|
||||
|
||||
@Override
|
||||
public synchronized void recordOutput(
|
||||
String policyId, String identity, String gate, String contentHash) {
|
||||
assertFalse(
|
||||
Files.exists(Path.of(identity)),
|
||||
"output must be recorded before it is visible at its final path");
|
||||
recorded = true;
|
||||
super.recordOutput(policyId, identity, gate, contentHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+6
-4
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.trigger;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -26,6 +27,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
@@ -100,8 +102,8 @@ class FolderWatchTriggerTest {
|
||||
|
||||
trigger.runForChangedDirs(Set.of(normalized("/in/a")));
|
||||
|
||||
verify(policyRunner).run(a);
|
||||
verify(policyRunner, never()).run(b);
|
||||
verify(policyRunner).run(a, SweepKind.LIGHT);
|
||||
verify(policyRunner, never()).run(eq(b), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -112,8 +114,8 @@ class FolderWatchTriggerTest {
|
||||
|
||||
trigger.runForChangedDirs(Set.of(normalized("/in/a")));
|
||||
|
||||
verify(policyRunner).run(good);
|
||||
verify(policyRunner, never()).run(bad);
|
||||
verify(policyRunner).run(good, SweepKind.LIGHT);
|
||||
verify(policyRunner, never()).run(eq(bad), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+4
-1
@@ -26,6 +26,7 @@ import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.DatabaseServiceInterface;
|
||||
import stirling.software.proprietary.security.service.SaveUserRequest;
|
||||
import stirling.software.proprietary.security.service.TeamMembershipService;
|
||||
import stirling.software.proprietary.security.service.TeamService;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
import stirling.software.proprietary.service.UserLicenseSettingsService;
|
||||
@@ -38,6 +39,7 @@ class InitialSecuritySetupTest {
|
||||
@Mock private DatabaseServiceInterface databaseService;
|
||||
@Mock private UserLicenseSettingsService licenseSettingsService;
|
||||
@Mock private Environment environment;
|
||||
@Mock private TeamMembershipService teamMembershipService;
|
||||
|
||||
private ApplicationProperties applicationProperties;
|
||||
private InitialSecuritySetup initialSecuritySetup;
|
||||
@@ -63,7 +65,8 @@ class InitialSecuritySetupTest {
|
||||
applicationProperties,
|
||||
databaseService,
|
||||
licenseSettingsService,
|
||||
environment);
|
||||
environment,
|
||||
teamMembershipService);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user