diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index d2f0472510..6fbebbbd68 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -308,6 +308,97 @@ public class ApplicationProperties { * explicitly requests it via {@code AiEngineClient.postWithTimeout}. */ private int longRunningTimeoutSeconds = 600; + + /** Timeout (seconds) for the SSE stream held open by long-running orchestrator runs. */ + private int streamTimeoutSeconds = 1800; + + /** Model + provider selection, forwarded to the engine per-request. */ + private Models models = new Models(); + + /** Retrieval-augmented-generation (RAG) knobs, forwarded to the engine per-request. */ + private Rag rag = new Rag(); + + /** Request size / cost guardrails. */ + private Limits limits = new Limits(); + + /** Per-capability on/off switches so an admin can disable individual AI tools. */ + private Features features = new Features(); + + @Data + public static class Models { + /** Provider driving the model strings: 'anthropic', 'openai', 'ollama', or 'custom'. */ + private String provider = "anthropic"; + + /** High-quality tier model name (without provider prefix), e.g. 'claude-haiku-4-5'. */ + private String smartModel = "claude-haiku-4-5"; + + /** Cheap/fast tier model name (without provider prefix). */ + private String fastModel = "claude-haiku-4-5"; + + private int smartMaxTokens = 8192; + private int fastMaxTokens = 2048; + + /** + * API key for the selected provider. Secret - masked in the admin API and overridable + * by the engine's own provider env var (e.g. ANTHROPIC_API_KEY) in environment-driven + * deployments. Empty means the engine falls back to its native env credentials. + */ + private String apiKey = ""; + + /** + * OpenAI-compatible base URL for 'ollama' / 'custom' providers (e.g. + * http://ollama:11434/v1). Ignored for anthropic/openai. SSRF-sensitive - admin only. + */ + private String baseUrl = ""; + } + + @Data + public static class Rag { + /** + * Embedding provider: 'voyageai', 'openai', 'ollama', or 'custom' (OpenAI-compatible). + */ + private String embeddingProvider = "voyageai"; + + /** Embedding model name (without provider prefix), e.g. 'voyage-4'. */ + private String embeddingModel = "voyage-4"; + + /** + * Secret API key for the embedding provider; masked + env-overridable like + * models.apiKey. + */ + private String embeddingApiKey = ""; + + /** + * OpenAI-compatible base URL for 'ollama' / 'custom' embedding providers (e.g. + * http://ollama:11434/v1). Ignored for voyageai/openai. SSRF-sensitive - admin only. + */ + private String embeddingBaseUrl = ""; + + /** How many chunks retrieval returns per search. */ + private int topK = 20; + + /** Per-run cap on knowledge-search tool calls before the agent must answer. */ + private int maxSearches = 5; + } + + @Data + public static class Limits { + private int maxPages = 200; + private int maxCharacters = 200000; + + /** Process-wide cap on concurrent model API calls (engine restart to apply). */ + private int modelMaxConcurrency = 32; + } + + @Data + public static class Features { + private boolean chat = true; + private boolean documentQuestions = true; + private boolean createPdf = true; + private boolean mathAuditor = true; + private boolean pdfComment = true; + private boolean classify = true; + } } /** diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/ConfigController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/ConfigController.java index 27477f9c2a..8044050025 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/ConfigController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/ConfigController.java @@ -336,7 +336,19 @@ public class ConfigController { configData.put("premiumEnabled", applicationProperties.getPremium().isEnabled()); // AI Engine settings - configData.put("aiEngineEnabled", applicationProperties.getAiEngine().isEnabled()); + ApplicationProperties.AiEngine aiEngineConfig = applicationProperties.getAiEngine(); + configData.put("aiEngineEnabled", aiEngineConfig.isEnabled()); + // Per-capability flags let the UI hide individual AI tools an admin has turned off. + ApplicationProperties.AiEngine.Features aiFeatures = aiEngineConfig.getFeatures(); + configData.put( + "aiFeatures", + Map.ofEntries( + Map.entry("chat", aiFeatures.isChat()), + Map.entry("documentQuestions", aiFeatures.isDocumentQuestions()), + Map.entry("createPdf", aiFeatures.isCreatePdf()), + Map.entry("mathAuditor", aiFeatures.isMathAuditor()), + Map.entry("pdfComment", aiFeatures.isPdfComment()), + Map.entry("classify", aiFeatures.isClassify()))); // Timestamp TSA settings — single source of truth for presets + admin URLs ApplicationProperties.Security.Timestamp tsConfig = diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index 85d97158b3..a5db905103 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -366,6 +366,34 @@ aiEngine: enabled: false # Set to 'true' to enable the AI engine integration url: http://localhost:5001 # URL of the Python AI engine timeoutSeconds: 120 # Timeout in seconds for AI engine requests + longRunningTimeoutSeconds: 600 # Timeout (seconds) for heavy operations like RAG ingestion of large documents + streamTimeoutSeconds: 1800 # SSE stream timeout (seconds) for long-running orchestrator runs + models: + provider: anthropic # Model provider: 'anthropic', 'openai', 'ollama', or 'custom' (OpenAI-compatible) + smartModel: claude-haiku-4-5 # High-quality tier model name (no provider prefix) + fastModel: claude-haiku-4-5 # Cheap/fast tier model name (no provider prefix) + smartMaxTokens: 8192 # Max output tokens for the smart tier + fastMaxTokens: 2048 # Max output tokens for the fast tier + apiKey: "" # API key for the selected provider (secret). Empty = engine uses its native env credentials (e.g. ANTHROPIC_API_KEY) + baseUrl: "" # OpenAI-compatible base URL for 'ollama'/'custom' providers (e.g. http://ollama:11434/v1). Ignored for anthropic/openai + rag: + embeddingProvider: voyageai # Embedding provider: 'voyageai', 'openai', 'ollama', or 'custom' (OpenAI-compatible) + embeddingModel: voyage-4 # Embedding model name (no provider prefix) + embeddingApiKey: "" # Secret API key for the embedding provider. Empty = engine uses its native env credentials (e.g. VOYAGE_API_KEY) + embeddingBaseUrl: "" # OpenAI-compatible base URL for 'ollama'/'custom' embedding providers (e.g. http://ollama:11434/v1). Ignored for voyageai/openai + topK: 20 # Number of chunks retrieval returns per search + maxSearches: 5 # Per-run cap on knowledge-search tool calls before the agent must answer + limits: + maxPages: 200 # Upper bound on PDF pages the engine will process per request + maxCharacters: 200000 # Upper bound on characters of extracted text per request + modelMaxConcurrency: 32 # Process-wide cap on concurrent model API calls (engine restart to apply) + features: # Per-capability switches; turn an individual AI tool off without disabling the whole engine + chat: true # Assistant chat + documentQuestions: true # Ask-questions-about-a-PDF + createPdf: true # Generate a PDF from a natural-language spec + mathAuditor: true # Numerical/formula contradiction auditing + pdfComment: true # AI-authored PDF comments/annotations + classify: true # Automatic document classification/labelling policies: # Folder automations can read from and write to the directories you allow here, so treat this as a @@ -385,6 +413,8 @@ policies: mcp: enabled: false # Master switch. 'false' (default) means no /mcp endpoint, no metadata, no beans wired. scopesEnabled: true # Enforce mcp.tools.read / mcp.tools.write scopes derived from operation category + maxRequestBytes: 10485760 # Max size (bytes) of an incoming MCP tool request payload (default 10 MB) + maxInlineResponseBytes: 10485760 # Max size (bytes) of an MCP tool response returned inline before it is rejected (default 10 MB) allowedOperations: [] # Tool allow-list (operation ids, e.g. ['compress-pdf']). Empty = all. When set, ONLY these are exposed over MCP. blockedOperations: [] # Tool deny-list (operation ids). Always removed from MCP even if otherwise allowed. auth: diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/AiEngineController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/AiEngineController.java index a96b3d1240..d59b3a8872 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/AiEngineController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/AiEngineController.java @@ -7,7 +7,6 @@ import java.util.concurrent.Executor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -28,6 +27,7 @@ import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.job.ResultFile; import stirling.software.common.service.JobOwnershipService; import stirling.software.common.service.TaskManager; @@ -63,12 +63,11 @@ public class AiEngineController { private final UserServiceInterface userService; /** - * SSE emitter timeout. Long enough to accommodate multi-gigabyte PDF workflows (OCR on a + * SSE emitter timeout in ms. Long enough to accommodate multi-gigabyte PDF workflows (OCR on a * 1000-page scan, splitting a huge PDF, etc.) without the emitter completing out from under the - * executor. Configurable via {@code stirling.ai.streamTimeoutMs}. + * executor. Derived from {@code aiEngine.streamTimeoutSeconds}. */ - @Value("${stirling.ai.streamTimeoutMs:1800000}") - private long streamTimeoutMs; + private final long streamTimeoutMs; public AiEngineController( AiEngineClient aiEngineClient, @@ -78,6 +77,7 @@ public class AiEngineController { TaskManager taskManager, JobOwnershipService jobOwnershipService, AiEngineEndpointResolver endpointResolver, + ApplicationProperties applicationProperties, @Autowired(required = false) UserServiceInterface userService) { this.aiEngineClient = aiEngineClient; this.aiWorkflowService = aiWorkflowService; @@ -87,6 +87,8 @@ public class AiEngineController { this.jobOwnershipService = jobOwnershipService; this.endpointResolver = endpointResolver; this.userService = userService; + this.streamTimeoutMs = + applicationProperties.getAiEngine().getStreamTimeoutSeconds() * 1000L; } private String currentUserId() { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java index 629a377cf3..f5076cf3cf 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/ClassifyLabelController.java @@ -36,6 +36,7 @@ import stirling.software.proprietary.classification.store.ClassificationLabelSto import stirling.software.proprietary.model.api.ai.AiPageText; import stirling.software.proprietary.policy.config.PolicyManagementAuthority; import stirling.software.proprietary.service.AiEngineClient; +import stirling.software.proprietary.service.AiFeatureGate; import stirling.software.proprietary.service.PdfContentExtractor; import tools.jackson.databind.JsonNode; @@ -68,6 +69,7 @@ public class ClassifyLabelController { private final PdfContentExtractor pdfContentExtractor; private final PdfMetadataService pdfMetadataService; private final AiEngineClient aiEngineClient; + private final AiFeatureGate aiFeatureGate; private final ObjectMapper objectMapper; private final UserServiceInterface userService; @@ -86,6 +88,7 @@ public class ClassifyLabelController { PdfContentExtractor pdfContentExtractor, PdfMetadataService pdfMetadataService, AiEngineClient aiEngineClient, + AiFeatureGate aiFeatureGate, ObjectMapper objectMapper, @Autowired(required = false) UserServiceInterface userService, @Autowired(required = false) ClassificationLabelStore labelStore, @@ -95,6 +98,7 @@ public class ClassifyLabelController { this.pdfContentExtractor = pdfContentExtractor; this.pdfMetadataService = pdfMetadataService; this.aiEngineClient = aiEngineClient; + this.aiFeatureGate = aiFeatureGate; this.objectMapper = objectMapper; this.userService = userService; this.labelStore = labelStore; @@ -111,6 +115,7 @@ public class ClassifyLabelController { + " intended for direct client use.") public ResponseEntity classifyAndLabel( @RequestParam("fileInput") MultipartFile fileInput) throws IOException { + aiFeatureGate.requireClassify(); try (PDDocument document = pdfDocumentFactory.load(fileInput, true)) { String fileName = safeFileName(fileInput.getOriginalFilename()); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AdminSettingsController.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AdminSettingsController.java index 14ddde602a..fa1571fb73 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AdminSettingsController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AdminSettingsController.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,6 +46,7 @@ import stirling.software.common.util.RegexPatternUtils; import stirling.software.proprietary.security.model.api.admin.SettingValueResponse; import stirling.software.proprietary.security.model.api.admin.UpdateSettingValueRequest; import stirling.software.proprietary.security.model.api.admin.UpdateSettingsRequest; +import stirling.software.proprietary.service.AiEngineConfigSync; import tools.jackson.core.type.TypeReference; import tools.jackson.databind.ObjectMapper; @@ -58,6 +60,7 @@ public class AdminSettingsController { private final ApplicationProperties applicationProperties; private final ObjectMapper objectMapper; private final ApplicationContext applicationContext; + private final AiEngineConfigSync aiEngineConfigSync; // Track settings that have been modified but not yet applied (require restart) private static final ConcurrentHashMap pendingChanges = @@ -172,6 +175,28 @@ public class AdminSettingsController { .body(Map.of("error", "No settings provided to update")); } + // Work on a mutable copy so the removal below works even if the caller passed an + // immutable map, then drop masked sensitive values so a UI round-trip never overwrites + // a real secret (e.g. an API key) with the "********" placeholder from the GET + // response. + settings = new LinkedHashMap<>(settings); + settings.entrySet() + .removeIf( + e -> { + if (!"********".equals(e.getValue())) { + return false; + } + String key = e.getKey(); + String leaf = + key.contains(".") + ? key.substring(key.lastIndexOf('.') + 1) + : key; + return isSensitiveFieldWithPath(leaf, key); + }); + if (settings.isEmpty()) { + return ResponseEntity.ok(Map.of("message", "No changed settings to update.")); + } + // Validate all settings first before applying any changes for (Map.Entry entry : settings.entrySet()) { String key = entry.getKey(); @@ -206,6 +231,10 @@ public class AdminSettingsController { pendingChanges.put(key, value != null ? value : ""); } + // If AI settings changed, push them to the engine live so model/RAG/limit changes + // apply without waiting for a processor restart. + maybePushAiEngineLive(); + return ResponseEntity.ok( Map.of( "message", @@ -600,6 +629,23 @@ public class AdminSettingsController { } } + /** + * Forward any pending {@code aiEngine.*} changes to the AI engine immediately. The running bean + * overlaid with these pending values is the current settings.yml state; {@link + * AiEngineConfigSync} no-ops unless AI is enabled and an engine-relevant key changed. + */ + private void maybePushAiEngineLive() { + Map aiEnginePending = new HashMap<>(); + for (Map.Entry entry : pendingChanges.entrySet()) { + if (entry.getKey().startsWith("aiEngine.")) { + aiEnginePending.put(entry.getKey(), entry.getValue()); + } + } + if (!aiEnginePending.isEmpty()) { + aiEngineConfigSync.pushLiveAfterSave(aiEnginePending); + } + } + private Object getSectionData(String sectionName) { if (sectionName == null || sectionName.trim().isEmpty()) { return null; @@ -843,8 +889,13 @@ public class AdminSettingsController { return true; } - // Check for fields containing 'password' or 'secret' - return lowerField.contains("password") || lowerField.contains("secret"); + // Substring match for secret-bearing field names. Covers provider credentials such as + // aiEngine.models.apiKey and aiEngine.rag.embeddingApiKey (which the exact-name set would + // miss) so keys are never returned to the client in cleartext. + return lowerField.contains("password") + || lowerField.contains("secret") + || lowerField.contains("apikey") + || lowerField.contains("token"); } /** Create a masked representation for sensitive fields */ diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiEngineConfigSync.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiEngineConfigSync.java new file mode 100644 index 0000000000..30bb088400 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiEngineConfigSync.java @@ -0,0 +1,171 @@ +package stirling.software.proprietary.service; + +import java.util.Map; + +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.ApplicationProperties.AiEngine; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ObjectNode; + +/** + * Pushes the admin-configured AI settings (model provider/name, per-provider API key, RAG choices, + * limits) to the Python engine so a self-hosted deployment can drive the engine's model and API key + * from the Stirling settings UI. Pushed on processor startup and again live whenever AI settings + * are saved (so model/RAG/limit changes apply without a restart). The engine applies it live + * (rebuilds its models) and caches it, so it self-restores on its own reboot. Empty + * key/baseUrl/model fields mean "keep the engine's own environment credential", so + * environment-driven deployments (where the engine sets {@code STIRLING_ALLOW_CONFIG_PUSH=false} + * and rejects the push) stay fully env-controlled. Best-effort and non-blocking: a slow or + * unreachable engine never delays or fails Stirling startup. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class AiEngineConfigSync { + + private static final int MAX_ATTEMPTS = 5; + private static final long RETRY_DELAY_MS = 3000L; + + private final ApplicationProperties applicationProperties; + private final AiEngineClient aiEngineClient; + private final ObjectMapper objectMapper; + + @EventListener(ApplicationReadyEvent.class) + public void pushConfigOnStartup() { + AiEngine cfg = applicationProperties.getAiEngine(); + if (!cfg.isEnabled()) { + return; + } + // Engine may still be booting; push on a virtual thread with a few retries so we never + // block or crash Stirling startup when the engine is slow or briefly unreachable. + Thread.ofVirtual().name("ai-engine-config-sync").start(() -> pushWithRetries(cfg)); + } + + /** + * Push AI settings to the engine immediately after an admin save so model/RAG/limit changes + * reach the engine without waiting for a processor restart. {@code pendingAiEngine} are the + * pending {@code aiEngine.*} dot-notation changes; the running bean overlaid with these is the + * current settings.yml state. No-op unless AI is enabled and an engine-relevant key changed. + */ + public void pushLiveAfterSave(Map pendingAiEngine) { + // Gate on the RUNNING bean: AiEngineClient refuses calls while the bean is disabled, so + // pushing on a pending-but-not-restarted enable would always fail. The post-restart + // startup push covers first-time enablement. + if (pendingAiEngine == null + || pendingAiEngine.isEmpty() + || !applicationProperties.getAiEngine().isEnabled()) { + return; + } + boolean engineRelevant = + pendingAiEngine.keySet().stream().anyMatch(AiEngineConfigSync::isEngineRelevantKey); + if (!engineRelevant) { + return; + } + ObjectNode node = buildConfigNode(applicationProperties.getAiEngine()); + pendingAiEngine.forEach((k, v) -> overlayIfEngineRelevant(node, k, v)); + String body = node.toString(); + Thread.ofVirtual().name("ai-engine-config-live-push").start(() -> pushOnce(body)); + } + + private void pushOnce(String body) { + try { + aiEngineClient.post("/api/v1/config", body, null); + log.info("Pushed AI engine configuration after settings change"); + } catch (Exception e) { + log.warn( + "Live AI engine config push failed: {} (will re-sync on next restart)", + e.getMessage()); + } + } + + // Only models/rag/limits are forwarded to the engine; enabled/url/timeouts/features are + // processor-side and don't need a live engine push. + private static boolean isEngineRelevantKey(String key) { + return key.startsWith("aiEngine.models.") + || key.startsWith("aiEngine.rag.") + || key.startsWith("aiEngine.limits."); + } + + private void overlayIfEngineRelevant(ObjectNode node, String key, Object value) { + if (!isEngineRelevantKey(key)) { + return; + } + String[] parts = key.substring("aiEngine.".length()).split("\\."); + ObjectNode parent = node; + for (int i = 0; i < parts.length - 1; i++) { + JsonNode child = parent.get(parts[i]); + parent = (child instanceof ObjectNode on) ? on : parent.putObject(parts[i]); + } + parent.set(parts[parts.length - 1], objectMapper.valueToTree(value)); + } + + private void pushWithRetries(AiEngine cfg) { + String body = buildConfigNode(cfg).toString(); + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + aiEngineClient.post("/api/v1/config", body, null); + log.info("Pushed AI engine configuration on startup (attempt {})", attempt); + return; + } catch (Exception e) { + log.warn( + "AI engine config push failed (attempt {}/{}): {}", + attempt, + MAX_ATTEMPTS, + e.getMessage()); + if (attempt < MAX_ATTEMPTS) { + try { + Thread.sleep(RETRY_DELAY_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } + } + log.warn( + "Giving up pushing AI engine configuration after {} attempts; the engine will use" + + " its own environment configuration until the next restart.", + MAX_ATTEMPTS); + } + + private ObjectNode buildConfigNode(AiEngine cfg) { + AiEngine.Models m = cfg.getModels(); + AiEngine.Rag r = cfg.getRag(); + AiEngine.Limits l = cfg.getLimits(); + + ObjectNode root = objectMapper.createObjectNode(); + + ObjectNode models = root.putObject("models"); + models.put("provider", m.getProvider()); + models.put("smartModel", m.getSmartModel()); + models.put("fastModel", m.getFastModel()); + models.put("smartMaxTokens", m.getSmartMaxTokens()); + models.put("fastMaxTokens", m.getFastMaxTokens()); + models.put("apiKey", m.getApiKey()); + models.put("baseUrl", m.getBaseUrl()); + + ObjectNode rag = root.putObject("rag"); + rag.put("embeddingProvider", r.getEmbeddingProvider()); + rag.put("embeddingModel", r.getEmbeddingModel()); + rag.put("embeddingApiKey", r.getEmbeddingApiKey()); + rag.put("embeddingBaseUrl", r.getEmbeddingBaseUrl()); + rag.put("topK", r.getTopK()); + rag.put("maxSearches", r.getMaxSearches()); + + ObjectNode limits = root.putObject("limits"); + limits.put("maxPages", l.getMaxPages()); + limits.put("maxCharacters", l.getMaxCharacters()); + limits.put("modelMaxConcurrency", l.getModelMaxConcurrency()); + + return root; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiFeatureGate.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiFeatureGate.java new file mode 100644 index 0000000000..b77e2b6e44 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiFeatureGate.java @@ -0,0 +1,58 @@ +package stirling.software.proprietary.service; + +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.server.ResponseStatusException; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.ApplicationProperties.AiEngine.Features; + +/** + * Central gate for the per-capability AI feature switches ({@code aiEngine.features.*}). Each AI + * capability entry point calls the matching {@code require*} method so an admin who turns a feature + * off in the settings UI gets a clean 503 instead of the request silently reaching the engine. All + * checks also fail closed when the AI engine itself is disabled. + */ +@Component +@RequiredArgsConstructor +public class AiFeatureGate { + + private final ApplicationProperties applicationProperties; + + private Features features() { + return applicationProperties.getAiEngine().getFeatures(); + } + + private void require(boolean featureEnabled, String feature) { + if (!applicationProperties.getAiEngine().isEnabled() || !featureEnabled) { + throw new ResponseStatusException( + HttpStatus.SERVICE_UNAVAILABLE, "AI feature '" + feature + "' is disabled"); + } + } + + public void requireChat() { + require(features().isChat(), "chat"); + } + + public void requireDocumentQuestions() { + require(features().isDocumentQuestions(), "documentQuestions"); + } + + public void requireCreatePdf() { + require(features().isCreatePdf(), "createPdf"); + } + + public void requireMathAuditor() { + require(features().isMathAuditor(), "mathAuditor"); + } + + public void requirePdfComment() { + require(features().isPdfComment(), "pdfComment"); + } + + public void requireClassify() { + require(features().isClassify(), "classify"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java index c5ee39be14..cc47cb36c8 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/controller/api/ClassifyLabelControllerTest.java @@ -32,6 +32,7 @@ import stirling.software.proprietary.classification.model.ClassificationLabels; import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore; import stirling.software.proprietary.policy.config.PolicyManagementAuthority; import stirling.software.proprietary.service.AiEngineClient; +import stirling.software.proprietary.service.AiFeatureGate; import stirling.software.proprietary.service.PdfContentExtractor; import tools.jackson.databind.JsonNode; @@ -49,6 +50,7 @@ class ClassifyLabelControllerTest { @Mock private PdfContentExtractor pdfContentExtractor; @Mock private PdfMetadataService pdfMetadataService; @Mock private AiEngineClient aiEngineClient; + @Mock private AiFeatureGate aiFeatureGate; @Mock private PolicyManagementAuthority policyManagementAuthority; private final ObjectMapper objectMapper = JsonMapper.builder().build(); @@ -65,6 +67,7 @@ class ClassifyLabelControllerTest { pdfContentExtractor, pdfMetadataService, aiEngineClient, + aiFeatureGate, objectMapper, null, labelStore, diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AdminSettingsControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AdminSettingsControllerTest.java index 5dc0e92963..92d10bb907 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AdminSettingsControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/AdminSettingsControllerTest.java @@ -26,6 +26,7 @@ import stirling.software.common.util.GeneralUtils; import stirling.software.proprietary.security.model.api.admin.SettingValueResponse; import stirling.software.proprietary.security.model.api.admin.UpdateSettingValueRequest; import stirling.software.proprietary.security.model.api.admin.UpdateSettingsRequest; +import stirling.software.proprietary.service.AiEngineConfigSync; import tools.jackson.databind.ObjectMapper; import tools.jackson.databind.json.JsonMapper; @@ -37,6 +38,7 @@ class AdminSettingsControllerTest { private ApplicationProperties applicationProperties; private ObjectMapper objectMapper; private ApplicationContext applicationContext; + private AiEngineConfigSync aiEngineConfigSync; private AdminSettingsController controller; @@ -45,9 +47,13 @@ class AdminSettingsControllerTest { applicationProperties = new ApplicationProperties(); objectMapper = JsonMapper.builder().build(); applicationContext = org.mockito.Mockito.mock(ApplicationContext.class); + aiEngineConfigSync = org.mockito.Mockito.mock(AiEngineConfigSync.class); controller = new AdminSettingsController( - applicationProperties, objectMapper, applicationContext); + applicationProperties, + objectMapper, + applicationContext, + aiEngineConfigSync); clearPendingChanges(); } diff --git a/engine/.env b/engine/.env index a99abf0ccf..d3898eb612 100644 --- a/engine/.env +++ b/engine/.env @@ -88,3 +88,8 @@ STIRLING_LOG_FILE= # Use when diagnosing worker stalls: a hung call shows a "Request" line with # no matching "Response" line. Noisy; leave off in normal use. STIRLING_HTTP_DEBUG=false + +# Allow the Java processor to push admin-configured AI settings (model, +# credentials, limits) to POST /api/v1/config at startup. Set false in +# environment-driven deployments so the environment is the single source of truth. +#STIRLING_ALLOW_CONFIG_PUSH=true diff --git a/engine/pyproject.toml b/engine/pyproject.toml index fa972c8495..dd933244be 100644 --- a/engine/pyproject.toml +++ b/engine/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" description = "AI Document Engine" requires-python = ">=3.13" dependencies = [ + "cryptography>=44.0.0", "fastapi>=0.116.0", "jinja2>=3.1.0", "pgvector>=0.3.6", diff --git a/engine/src/stirling/agents/math_presentation.py b/engine/src/stirling/agents/math_presentation.py index bfb6fa7ef8..d8ec5b9d12 100644 --- a/engine/src/stirling/agents/math_presentation.py +++ b/engine/src/stirling/agents/math_presentation.py @@ -15,6 +15,7 @@ from __future__ import annotations from pydantic import Field from pydantic_ai import Agent +from stirling.agents.output_mode import output_retries from stirling.contracts import ( MathAuditorToolReportArtifact, OrchestratorRequest, @@ -68,6 +69,10 @@ class MathIntentClassifier: self._agent: Agent[None, _MathIntentDecision] = Agent( model=runtime.fast_model, output_type=_MathIntentDecision, + # Local models emit valid structured output only intermittently (they + # sometimes wrap the JSON in prose); a few extra retries make this + # hot-path classifier reliable. No-op for real providers. + retries=output_retries(runtime.settings.chat_provider), system_prompt=_MATH_INTENT_SYSTEM_PROMPT, model_settings=runtime.fast_model_settings, ) diff --git a/engine/src/stirling/agents/orchestrator.py b/engine/src/stirling/agents/orchestrator.py index e07e771e34..93ff751221 100644 --- a/engine/src/stirling/agents/orchestrator.py +++ b/engine/src/stirling/agents/orchestrator.py @@ -2,12 +2,14 @@ from __future__ import annotations import logging from dataclasses import dataclass -from typing import assert_never +from typing import Literal, assert_never +from pydantic import ConfigDict, Field from pydantic_ai import Agent -from pydantic_ai.output import ToolOutput +from pydantic_ai.output import NativeOutput, ToolOutput from pydantic_ai.tools import RunContext +from stirling.agents.output_mode import output_retries, uses_tool_output from stirling.agents.pdf_create import PdfCreateAgent from stirling.agents.pdf_edit import PdfEditAgent from stirling.agents.pdf_questions import PdfQuestionAgent @@ -27,6 +29,7 @@ from stirling.contracts import ( format_file_names, ) from stirling.contracts.pdf_create import PdfCreateOrchestrateResponse +from stirling.models import ApiModel from stirling.services import AppRuntime logger = logging.getLogger(__name__) @@ -38,6 +41,38 @@ class OrchestratorDeps: request: OrchestratorRequest +# Enum-style routing used for Ollama/custom (OpenAI-compatible local) models. The +# function-ToolOutput delegates below take no arguments (they read everything from +# ctx), but local models reliably try to pass the user's message as tool args, which +# the zero-arg delegate schemas reject. Having the model pick a capability by name and +# dispatching in Python (via the same _run_* methods used on resume) sidesteps that. +_RouteCapability = Literal["pdf_edit", "pdf_question", "user_spec", "pdf_review", "pdf_create", "unsupported"] + + +class _RouteDecision(ApiModel): + # Local models routinely add stray tool args (the filename, the question echoed back) + # and send null for optional fields; tolerate both so routing never fails on cosmetics. + model_config = ConfigDict(extra="ignore") + capability: _RouteCapability + message: str | None = Field( + default=None, + description="Only for capability='unsupported': a short, helpful message to show the user.", + ) + + +_ROUTER_SYSTEM_PROMPT = ( + "You are the top-level router. Choose exactly one capability that best handles the request:\n" + "- pdf_edit: modify or convert one or more attached PDFs.\n" + "- pdf_question: answer questions about the contents of the attached PDFs.\n" + "- user_spec: create or define an agent spec.\n" + "- pdf_review: return the PDF with review comments/annotations attached.\n" + "- pdf_create: generate a NEW document from scratch (invoice, report, letter) - no input file.\n" + "- unsupported: none of the above fit, or the user asks about the assistant itself; put a " + "helpful message in 'message'.\n" + "Respond with the capability and (only for unsupported) a message." +) + + class OrchestratorAgent: def __init__(self, runtime: AppRuntime) -> None: self.runtime = runtime @@ -86,6 +121,9 @@ class OrchestratorAgent: description="Return this when none of the delegate outputs fit the request.", ), ], + # Local models (Ollama/custom) pick a delegate less reliably; give the + # routing a few extra validation retries. No-op for real providers. + retries=output_retries(runtime.settings.chat_provider), deps_type=OrchestratorDeps, system_prompt=( "You are the top-level orchestrator. " @@ -103,6 +141,24 @@ class OrchestratorAgent: ), model_settings=runtime.fast_model_settings, ) + # Local models can't drive the zero-arg function-delegate routing above, so + # route them by capability name and dispatch in Python instead. + self._route_via_enum = uses_tool_output(runtime.settings.chat_provider) + # The router has no tools of its own, so NativeOutput (a direct structured + # response) works on Ollama here - unlike tool-using agents, which need + # ToolOutput. A single output tool would tempt a local model to answer in + # plain text and never call it ("include your response in a tool call"). + self._router = ( + Agent( + model=runtime.fast_model, + output_type=NativeOutput([_RouteDecision]), + retries=output_retries(runtime.settings.chat_provider), + system_prompt=_ROUTER_SYSTEM_PROMPT, + model_settings=runtime.fast_model_settings, + ) + if self._route_via_enum + else None + ) async def handle(self, request: OrchestratorRequest) -> OrchestratorResponse: logger.info( @@ -114,6 +170,8 @@ class OrchestratorAgent: ) if request.resume_with is not None: return await self._resume(request, request.resume_with) + if self._router is not None: + return await self._route_and_dispatch(request) result = await self.agent.run( self._build_prompt(request), deps=OrchestratorDeps(runtime=self.runtime, request=request), @@ -121,6 +179,31 @@ class OrchestratorAgent: logger.info("[orchestrator] routed -> %s", type(result.output).__name__) return result.output + async def _route_and_dispatch(self, request: OrchestratorRequest) -> OrchestratorResponse: + """Local-model routing: pick a capability by name, then dispatch in Python.""" + assert self._router is not None + result = await self._router.run(self._build_prompt(request)) + decision = result.output + logger.info("[orchestrator] enum-routed -> %s", decision.capability) + match decision.capability: + case "pdf_edit": + return await self._run_pdf_edit(request) + case "pdf_question": + return await self._run_pdf_question(request) + case "user_spec": + return await self._run_agent_draft(request) + case "pdf_review": + return await self._run_pdf_review(request) + case "pdf_create": + return await self._run_pdf_create(request) + case "unsupported": + return UnsupportedCapabilityResponse( + capability="orchestrate", + message=decision.message or "I can't help with that request.", + ) + case _ as unreachable: + assert_never(unreachable) + async def _resume(self, request: OrchestratorRequest, capability: SupportedCapability) -> OrchestratorResponse: """Fast-path to get back to the correct endpoint without having to call AI. diff --git a/engine/src/stirling/agents/output_mode.py b/engine/src/stirling/agents/output_mode.py new file mode 100644 index 0000000000..f70bd9342e --- /dev/null +++ b/engine/src/stirling/agents/output_mode.py @@ -0,0 +1,46 @@ +"""Provider-aware structured-output strategy. + +OpenAI-compatible local models (Ollama, and 'custom' OpenAI-compatible endpoints) +block tool-calling whenever a native json-schema ``response_format`` is set: the +API constrains every completion to the output schema, so the model can never emit +a tool-call turn. Agents that combine structured output with tools (RAG search, +whole-document read, etc.) therefore get an ungrounded answer - the model fills the +schema directly instead of calling a retrieval tool. + +Delivering the structured result via a tool call (:class:`ToolOutput`) sidesteps +this: it's all tool-calling, which these endpoints handle, so the model can call a +retrieval tool and then the output tool. Real providers (Anthropic, OpenAI) keep +:class:`NativeOutput`, which is unaffected and preferred there. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from pydantic_ai.output import NativeOutput, ToolOutput + +# Providers whose OpenAI-compatible endpoint needs tool-delivered structured output. +_TOOL_OUTPUT_PROVIDERS = frozenset({"ollama", "custom"}) + + +def uses_tool_output(chat_provider: str) -> bool: + return chat_provider in _TOOL_OUTPUT_PROVIDERS + + +def structured_output(output_types: Sequence[Any], *, chat_provider: str) -> Any: + """Pick a structured-output spec compatible with the active chat provider. + + Ollama/custom deliver each output variant via a :class:`ToolOutput`; every other + provider uses a single :class:`NativeOutput` over the variants. + """ + types = list(output_types) + if uses_tool_output(chat_provider): + return [ToolOutput(t) for t in types] + return NativeOutput(types) + + +def output_retries(chat_provider: str, *, native: int = 1, tool: int = 6) -> int: + """Local models delivering via ToolOutput need more output-validation retries - + small models produce valid complex/nested structured output only intermittently.""" + return tool if uses_tool_output(chat_provider) else native diff --git a/engine/src/stirling/agents/pdf_questions.py b/engine/src/stirling/agents/pdf_questions.py index 5b956c468b..9bd349a7ec 100644 --- a/engine/src/stirling/agents/pdf_questions.py +++ b/engine/src/stirling/agents/pdf_questions.py @@ -3,10 +3,10 @@ from __future__ import annotations import logging from pydantic_ai import Agent -from pydantic_ai.output import NativeOutput from stirling.agents.contradiction import ContradictionCapability, ContradictionDetector from stirling.agents.math_presentation import MathIntentClassifier, extract_math_verdict +from stirling.agents.output_mode import output_retries, structured_output from stirling.agents.shared import ChunkedReasoner, WholeDocReaderCapability from stirling.contracts import ( AiFile, @@ -196,9 +196,17 @@ class PdfQuestionAgent: files=request.files, principals=principals, ) + # Ollama/custom (OpenAI-compatible local) models block tool-calling under a + # native json-schema response format, so deliver the structured result via a + # tool call instead - otherwise the model never calls the retrieval tools and + # answers ungrounded. Real providers keep NativeOutput. See agents.output_mode. + provider = self.runtime.settings.chat_provider agent = Agent( model=self.runtime.smart_model, - output_type=NativeOutput([PdfQuestionAnswerResponse, PdfQuestionNotFoundResponse]), + output_type=structured_output( + [PdfQuestionAnswerResponse, PdfQuestionNotFoundResponse], chat_provider=provider + ), + retries=output_retries(provider), system_prompt=PDF_QUESTION_SYSTEM_PROMPT, # pydantic-ai accepts a list of (string-or-callable) instruction sources; # it resolves each at run time and concatenates them for the model. diff --git a/engine/src/stirling/api/app.py b/engine/src/stirling/api/app.py index 34b0da36fd..d5df4e5407 100644 --- a/engine/src/stirling/api/app.py +++ b/engine/src/stirling/api/app.py @@ -3,28 +3,20 @@ from __future__ import annotations import asyncio import logging from contextlib import asynccontextmanager -from typing import Annotated -from fastapi import Depends, FastAPI +from fastapi import Depends, FastAPI, Request from pydantic_ai import Agent +from pydantic_ai.models import Model from pydantic_ai.models.instrumented import InstrumentationSettings -from stirling.agents import ( - DocumentClassifierAgent, - ExecutionPlanningAgent, - OrchestratorAgent, - PdfEditAgent, - PdfQuestionAgent, - UserSpecAgent, -) -from stirling.agents.ledger import MathAuditorAgent -from stirling.agents.pdf_comment import PdfCommentAgent +from stirling.api.bootstrap import apply_app_state, build_app_state from stirling.api.dependencies import enforce_required_user_id from stirling.api.engine_auth import EngineSharedSecretMiddleware from stirling.api.middleware import UserIdMiddleware from stirling.api.routes import ( agent_capabilities_router, agent_draft_router, + config_router, document_classifier_router, document_router, execution_router, @@ -34,10 +26,12 @@ from stirling.api.routes import ( pdf_edit_router, pdf_question_router, ) +from stirling.api.routes.config import CONFIG_APPLY_ERRORS, resolve_and_apply from stirling.config import AppSettings, load_settings +from stirling.config.config_cache import load_config from stirling.contracts import HealthResponse -from stirling.documents import DocumentService -from stirling.services import build_runtime, setup_posthog_tracking +from stirling.documents import DocumentService, EmbeddingService +from stirling.services import setup_posthog_tracking logger = logging.getLogger(__name__) @@ -83,22 +77,52 @@ def _load_startup_settings(fast_api: FastAPI) -> AppSettings: return load_settings() +def _restore_cached_config( + settings: AppSettings, +) -> tuple[AppSettings, Model | None, Model | None, EmbeddingService | None]: + """Restore the last-applied pushed config from the encrypted on-disk cache. + + Returns the effective settings plus the pre-built smart/fast models and + embedder to inject into the initial app state. Falls back to the env settings + (all-None) when config push is disabled, no cache exists, or the cached config + can't be applied (bad/unavailable model) - the cache never crashes boot. + """ + if not settings.allow_config_push: + return settings, None, None, None + cached = load_config() + if cached is None: + return settings, None, None, None + try: + effective, smart_model, fast_model, embedder, notes = resolve_and_apply(settings, cached) + except CONFIG_APPLY_ERRORS: + logger.warning("Cached AI config could not be applied; falling back to env settings", exc_info=True) + return settings, None, None, None + logger.info( + "Restored cached AI config: smart_model=%s fast_model=%s%s", + effective.smart_model_name, + effective.fast_model_name, + f"; {'; '.join(notes)}" if notes else "", + ) + return effective, smart_model, fast_model, embedder + + @asynccontextmanager async def lifespan(fast_api: FastAPI): # Load env vars on startup so we can immediately crash if required env vars aren't set settings = _load_startup_settings(fast_api) - runtime = build_runtime(settings) - fast_api.state.settings = settings - fast_api.state.runtime = runtime - fast_api.state.orchestrator_agent = OrchestratorAgent(runtime) - fast_api.state.pdf_edit_agent = PdfEditAgent(runtime) - fast_api.state.pdf_question_agent = PdfQuestionAgent(runtime) - fast_api.state.user_spec_agent = UserSpecAgent(runtime) - fast_api.state.execution_planning_agent = ExecutionPlanningAgent(runtime) - fast_api.state.math_auditor_agent = MathAuditorAgent(runtime) - fast_api.state.pdf_comment_agent = PdfCommentAgent(runtime) - fast_api.state.document_classifier_agent = DocumentClassifierAgent(runtime) - tracer_provider = setup_posthog_tracking(settings) + # Precedence: env < persisted cache < live push. Restore the last-applied pushed + # config unless config push is disabled (then env is the single source of truth). + effective, smart_model, fast_model, embedder = _restore_cached_config(settings) + app_state = build_app_state( + effective, + fast_model=fast_model, + smart_model=smart_model, + embedder=embedder, + ) + fast_api.state.settings = effective + apply_app_state(fast_api.state, app_state) + runtime = app_state.runtime + tracer_provider = setup_posthog_tracking(effective) if tracer_provider: Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider)) reaper_task = asyncio.create_task( @@ -135,10 +159,18 @@ app.include_router(ledger_router, dependencies=_user_gate) app.include_router(pdf_comments_router, dependencies=_user_gate) app.include_router(agent_capabilities_router, dependencies=_user_gate) app.include_router(document_classifier_router, dependencies=_user_gate) +# Config push is a system/admin sync from the Java processor with no X-User-Id, so +# it is guarded by the X-Engine-Auth shared secret (global middleware) and the +# allow_config_push flag only, deliberately NOT the per-user identity gate. +app.include_router(config_router) @app.get("/health", response_model=HealthResponse) -async def healthcheck(settings: Annotated[AppSettings, Depends(load_settings)]) -> HealthResponse: +async def healthcheck(http_request: Request) -> HealthResponse: + # Report the LIVE config (env < cache < push) held on app.state, not the + # boot-time env cache, so an admin "Test connection" check shows the model + # actually in use after a config push. Falls back to env if state isn't up yet. + settings: AppSettings = getattr(http_request.app.state, "settings", None) or load_settings() return HealthResponse( status="ok", smart_model=settings.smart_model_name, diff --git a/engine/src/stirling/api/bootstrap.py b/engine/src/stirling/api/bootstrap.py new file mode 100644 index 0000000000..68610327a8 --- /dev/null +++ b/engine/src/stirling/api/bootstrap.py @@ -0,0 +1,86 @@ +"""Assemble the runtime + every agent into a single swappable bundle. + +Both the lifespan startup and the config-push endpoint build the same object +graph: one :class:`AppRuntime` plus one agent instance per flow, each of which +captures ``runtime.smart_model``/``runtime.fast_model`` at construction. To change +models at runtime the whole bundle has to be rebuilt and swapped into +``app.state`` atomically, which is what :func:`build_app_state` / +:func:`apply_app_state` exist for. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import Any + +from pydantic_ai.models import Model + +from stirling.agents import ( + DocumentClassifierAgent, + ExecutionPlanningAgent, + OrchestratorAgent, + PdfEditAgent, + PdfQuestionAgent, + UserSpecAgent, +) +from stirling.agents.ledger import MathAuditorAgent +from stirling.agents.pdf_comment import PdfCommentAgent +from stirling.config import AppSettings +from stirling.documents import DocumentService, EmbeddingService +from stirling.services import AppRuntime, build_runtime + + +@dataclass(frozen=True) +class AppState: + """Every object the lifespan assigns onto ``fast_api.state``.""" + + runtime: AppRuntime + orchestrator_agent: OrchestratorAgent + pdf_edit_agent: PdfEditAgent + pdf_question_agent: PdfQuestionAgent + user_spec_agent: UserSpecAgent + execution_planning_agent: ExecutionPlanningAgent + math_auditor_agent: MathAuditorAgent + pdf_comment_agent: PdfCommentAgent + document_classifier_agent: DocumentClassifierAgent + + +def build_app_state( + settings: AppSettings, + *, + documents: DocumentService | None = None, + fast_model: Model | None = None, + smart_model: Model | None = None, + embedder: EmbeddingService | None = None, +) -> AppState: + """Build the runtime and every agent from ``settings``. + + The keyword arguments are threaded straight to :func:`build_runtime` so a + config-push rebuild can reuse the live document store and inject already + validated models (and, on a cache-restore boot, a pre-built embedder); all + None reproduces the original startup path. + """ + runtime = build_runtime( + settings, + documents=documents, + fast_model=fast_model, + smart_model=smart_model, + embedder=embedder, + ) + return AppState( + runtime=runtime, + orchestrator_agent=OrchestratorAgent(runtime), + pdf_edit_agent=PdfEditAgent(runtime), + pdf_question_agent=PdfQuestionAgent(runtime), + user_spec_agent=UserSpecAgent(runtime), + execution_planning_agent=ExecutionPlanningAgent(runtime), + math_auditor_agent=MathAuditorAgent(runtime), + pdf_comment_agent=PdfCommentAgent(runtime), + document_classifier_agent=DocumentClassifierAgent(runtime), + ) + + +def apply_app_state(state: Any, app_state: AppState) -> None: + """Copy every field of ``app_state`` onto a Starlette ``app.state`` object.""" + for field in fields(app_state): + setattr(state, field.name, getattr(app_state, field.name)) diff --git a/engine/src/stirling/api/routes/__init__.py b/engine/src/stirling/api/routes/__init__.py index f40e9918f8..440ff18f9d 100644 --- a/engine/src/stirling/api/routes/__init__.py +++ b/engine/src/stirling/api/routes/__init__.py @@ -1,5 +1,6 @@ from .agent_capabilities import router as agent_capabilities_router from .agent_drafts import router as agent_draft_router +from .config import router as config_router from .document_classifier import router as document_classifier_router from .documents import router as document_router from .execution import router as execution_router @@ -12,6 +13,7 @@ from .pdf_questions import router as pdf_question_router __all__ = [ "agent_capabilities_router", "agent_draft_router", + "config_router", "document_classifier_router", "document_router", "execution_router", diff --git a/engine/src/stirling/api/routes/config.py b/engine/src/stirling/api/routes/config.py new file mode 100644 index 0000000000..fa5ed203cd --- /dev/null +++ b/engine/src/stirling/api/routes/config.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import ipaddress +import logging + +from fastapi import APIRouter, HTTPException, Request, status +from openai import OpenAIError +from pydantic_ai.exceptions import UserError +from pydantic_ai.models import Model + +from stirling.api.bootstrap import apply_app_state, build_app_state +from stirling.config import AppSettings +from stirling.config.config_cache import save_config +from stirling.contracts import ConfigApplyResponse, ConfigPushRequest +from stirling.documents import EmbeddingService +from stirling.services import AppRuntime +from stirling.services.runtime import _build_model, validate_structured_output_support + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/config", tags=["config"]) + +# Model/provider construction + validation failures. The HTTP route maps these to a +# 400 (no swap); boot catches them to fall back to env when a cached config is bad. +CONFIG_APPLY_ERRORS = (ValueError, UserError, OpenAIError) + +_REINDEX_NOTE = ( + "Embedding model changed; existing indexed documents were embedded with the previous model and " + "must be re-indexed. If the embedding dimensionality changed, re-ingest before searching." +) + + +def _strip_provider_prefix(model_name: str) -> str: + """Drop a leading ``provider:`` from an env model string ("anthropic:x" -> "x"). + + Only used when falling back to the env-configured name under an explicit + pushed provider; a bare pushed name (which may itself contain ``:`` such as + "llama3:8b") is used verbatim and never passed through here. + """ + _, sep, rest = model_name.partition(":") + return rest if sep else model_name + + +def _compose_embedding_model(provider: str, model: str) -> str: + """Compose the engine's ``provider:model`` embedding string from pushed parts.""" + provider = provider.strip() + return f"{provider}:{model}" if provider else model + + +def _split_embedding_ref(ref: str) -> tuple[str, str]: + """Split an env embedding string ("voyageai:voyage-4") into (provider, model).""" + provider, sep, model = ref.partition(":") + return (provider, model) if sep else ("", ref) + + +def _keep(pushed: int | None, current: int) -> int: + """Return the pushed value, or the current one when the push omitted it.""" + return pushed if pushed is not None else current + + +def _is_loopback_client(request: Request) -> bool: + """True when the request originates from the local host (127.0.0.0/8, ::1).""" + client = request.client + if client is None: + return False + try: + return ipaddress.ip_address(client.host).is_loopback + except ValueError: + return client.host == "localhost" + + +def resolve_and_apply( + current: AppSettings, + request: ConfigPushRequest, +) -> tuple[AppSettings, Model, Model, EmbeddingService | None, list[str]]: + """Resolve a pushed config against the running settings. + + Builds and validates the smart/fast models, resolves the scalar overrides, + and builds a new embedder when the embedding config changed. Empty/None pushed + fields keep the current value. Returns the effective settings, the two built + models, an optional new embedder (None when the embedding config is unchanged), + and human-readable notes. + + Raises one of :data:`CONFIG_APPLY_ERRORS` if a chosen model/embedder fails to + build or validate; callers decide whether that becomes a 400 (HTTP route) or an + env fallback (boot). It never swaps any live state - the caller owns that. + """ + models = request.models + rag = request.rag + limits = request.limits + notes: list[str] = [] + + provider = models.provider.strip() + api_key = models.api_key + base_url = models.base_url + use_explicit_provider = bool(provider or api_key or base_url) + + if use_explicit_provider: + smart_name = models.smart_model or _strip_provider_prefix(current.smart_model_name) + fast_name = models.fast_model or _strip_provider_prefix(current.fast_model_name) + else: + # No provider/credentials pushed: keep the fully env-driven model strings. + smart_name = models.smart_model or current.smart_model_name + fast_name = models.fast_model or current.fast_model_name + + def _build(bare: str) -> Model: + if use_explicit_provider: + return _build_model(bare, provider=provider or None, api_key=api_key or None, base_url=base_url or None) + return _build_model(bare) + + smart_model = _build(smart_name) + fast_model = _build(fast_name) + validate_structured_output_support(smart_model, smart_name) + validate_structured_output_support(fast_model, fast_name) + + # Scalars: None / empty keep the current value. + smart_max_tokens = _keep(models.smart_max_tokens, current.smart_model_max_tokens) + fast_max_tokens = _keep(models.fast_max_tokens, current.fast_model_max_tokens) + top_k = _keep(rag.top_k, current.rag_default_top_k) + max_searches = _keep(rag.max_searches, current.rag_max_searches) + max_pages = _keep(limits.max_pages, current.max_pages) + max_characters = _keep(limits.max_characters, current.max_characters) + model_max_concurrency = _keep(limits.model_max_concurrency, current.model_max_concurrency) + + # Embedding: any non-empty embedding field triggers a rebuild; empty fields fall + # back to the running provider/model/creds so a partial push never clobbers env. + embedding_changed = bool( + rag.embedding_provider.strip() or rag.embedding_model.strip() or rag.embedding_api_key or rag.embedding_base_url + ) + rag_embedding_model = current.rag_embedding_model + new_embedder: EmbeddingService | None = None + if embedding_changed: + current_provider, current_model = _split_embedding_ref(current.rag_embedding_model) + embed_provider = rag.embedding_provider.strip() or current_provider + embed_model = rag.embedding_model.strip() or current_model + rag_embedding_model = _compose_embedding_model(embed_provider, embed_model) + new_embedder = EmbeddingService( + model_name=embed_model, + chunk_size=current.rag_chunk_size, + chunk_overlap=current.rag_chunk_overlap, + provider=embed_provider or None, + api_key=rag.embedding_api_key or None, + base_url=rag.embedding_base_url or None, + ) + notes.append(_REINDEX_NOTE) + + effective = current.model_copy( + update={ + "chat_provider": provider, + "smart_model_name": smart_name, + "fast_model_name": fast_name, + "smart_model_max_tokens": smart_max_tokens, + "fast_model_max_tokens": fast_max_tokens, + "rag_embedding_model": rag_embedding_model, + "rag_default_top_k": top_k, + "rag_max_searches": max_searches, + "max_pages": max_pages, + "max_characters": max_characters, + "model_max_concurrency": model_max_concurrency, + } + ) + return effective, smart_model, fast_model, new_embedder, notes + + +@router.post("", response_model=ConfigApplyResponse) +async def apply_config(request: ConfigPushRequest, http_request: Request) -> ConfigApplyResponse: + """Apply admin-pushed AI settings by rebuilding the runtime + agents in place. + + Gated by the X-Engine-Auth shared secret (global middleware) plus the + ``allow_config_push`` flag. Empty credential/model fields keep the engine's + env-configured values. Returns 403 when config push is disabled, 400 (without + swapping state) when a chosen model fails to build/validate. On success the + config is also persisted (encrypted) so it survives an engine restart. + """ + app = http_request.app + current: AppSettings = app.state.settings + if not current.allow_config_push: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Config push is disabled on this deployment (STIRLING_ALLOW_CONFIG_PUSH is false).", + ) + # Secure-by-default for this sensitive endpoint. When a shared secret is set the + # global middleware has already authenticated the caller (only the processor has the + # secret). When NO secret is set, only trust a loopback caller - a remote party must + # never be able to push a config unauthenticated, since a pushed base_url/model could + # repoint the engine to exfiltrate document content. + if not current.engine_shared_secret and not _is_loopback_client(http_request): + client_host = http_request.client.host if http_request.client else "unknown" + logger.warning("Rejected config push from non-local caller %s with no shared secret set", client_host) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=( + "Config push from a non-local caller requires STIRLING_ENGINE_SHARED_SECRET" + " to be set on both the engine and the processor." + ), + ) + + runtime: AppRuntime = app.state.runtime + try: + effective, smart_model, fast_model, new_embedder, notes = resolve_and_apply(current, request) + except CONFIG_APPLY_ERRORS as exc: + # Reject without touching the running config. + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + # Rebuild runtime + every agent, reusing the existing document store, then + # swap the whole bundle onto app.state so in-flight lookups see one config. + new_state = build_app_state( + effective, + documents=runtime.documents, + fast_model=fast_model, + smart_model=smart_model, + ) + app.state.settings = effective + apply_app_state(app.state, new_state) + # Retune retrieval breadth on the reused store without rebuilding it. + runtime.documents.default_top_k = effective.rag_default_top_k + if new_embedder is not None: + # Swap the embedder onto the reused DocumentService so we never tear down + # the live vector store / connection pool. + runtime.documents.embedder = new_embedder + + # Persist the applied config (encrypted) so it is restored on the next boot. + try: + save_config(request) + except OSError: + logger.warning("Applied AI config but failed to persist the encrypted cache", exc_info=True) + notes.append("Config applied but could not be persisted; it will not survive an engine restart.") + + logger.info( + "Applied pushed AI config: provider=%s smart_model=%s fast_model=%s top_k=%s", + request.models.provider.strip() or "", + effective.smart_model_name, + effective.fast_model_name, + effective.rag_default_top_k, + ) + + return ConfigApplyResponse( + status="applied", + provider=request.models.provider.strip(), + smart_model=effective.smart_model_name, + fast_model=effective.fast_model_name, + smart_max_tokens=effective.smart_model_max_tokens, + fast_max_tokens=effective.fast_model_max_tokens, + rag_embedding_model=effective.rag_embedding_model, + rag_top_k=effective.rag_default_top_k, + rag_max_searches=effective.rag_max_searches, + max_pages=effective.max_pages, + max_characters=effective.max_characters, + model_max_concurrency=effective.model_max_concurrency, + notes=notes, + ) diff --git a/engine/src/stirling/config/config_cache.py b/engine/src/stirling/config/config_cache.py new file mode 100644 index 0000000000..3086c84b97 --- /dev/null +++ b/engine/src/stirling/config/config_cache.py @@ -0,0 +1,114 @@ +"""Persistent, encrypted cache of the last-applied config-push. + +The engine otherwise holds pushed config in RAM only, so a restart reverts to the +env values until the processor pushes again. We persist the last-applied +:class:`ConfigPushRequest` to ``/ai_config_cache.enc`` (Fernet +encrypted) and reload it on boot so the effective config survives restarts. + +Key derivation: when ``STIRLING_ENGINE_SHARED_SECRET`` is set, the Fernet key is +HKDF-SHA256 over that secret (constant salt/info) so no key material touches disk. +Otherwise a random Fernet key is generated once and stored in a sibling +``ai_config_cache.key`` (0600, best-effort) with a one-time warning. +""" + +from __future__ import annotations + +import base64 +import logging +import os +import stat +from pathlib import Path + +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +from stirling.config.settings import ENGINE_ROOT, load_settings +from stirling.contracts import ConfigPushRequest + +logger = logging.getLogger(__name__) + +_CACHE_FILENAME = "ai_config_cache.enc" +_KEY_FILENAME = "ai_config_cache.key" +# Constant salt/info so the same shared secret always derives the same Fernet key. +_HKDF_SALT = b"stirling-ai-config-cache/v1/salt" +_HKDF_INFO = b"stirling-ai-config-cache/v1/fernet-key" + +_keyfile_warned = False + + +def _default_data_dir() -> Path: + """The engine data dir (where the sqlite store lives by default).""" + return ENGINE_ROOT / "data" + + +def _shared_secret() -> str: + return load_settings().engine_shared_secret + + +def _derive_key_from_secret(secret: str) -> bytes: + hkdf = HKDF(algorithm=hashes.SHA256(), length=32, salt=_HKDF_SALT, info=_HKDF_INFO) + return base64.urlsafe_b64encode(hkdf.derive(secret.encode("utf-8"))) + + +def _chmod_600(path: Path) -> None: + # Best-effort owner-only perms; silently ignored where unsupported (e.g. Windows). + try: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def _load_or_create_keyfile(data_dir: Path) -> bytes: + global _keyfile_warned + key_path = data_dir / _KEY_FILENAME + if key_path.exists(): + return key_path.read_bytes().strip() + key = Fernet.generate_key() + data_dir.mkdir(parents=True, exist_ok=True) + key_path.write_bytes(key) + _chmod_600(key_path) + if not _keyfile_warned: + logger.warning( + "STIRLING_ENGINE_SHARED_SECRET is not set; encrypting the AI config cache with a local" + " keyfile at %s (0600, best-effort). This is modest protection only - set a shared secret" + " for HKDF key derivation in any deployment where the cache must be strongly protected.", + key_path, + ) + _keyfile_warned = True + return key + + +def _fernet(data_dir: Path) -> Fernet: + secret = _shared_secret() + if secret: + return Fernet(_derive_key_from_secret(secret)) + return Fernet(_load_or_create_keyfile(data_dir)) + + +def save_config(request: ConfigPushRequest, *, data_dir: Path | None = None) -> None: + """Encrypt and persist the last-applied pushed config, overwriting any prior file.""" + data_dir = data_dir or _default_data_dir() + data_dir.mkdir(parents=True, exist_ok=True) + payload = request.model_dump_json(by_alias=True).encode("utf-8") + token = _fernet(data_dir).encrypt(payload) + (data_dir / _CACHE_FILENAME).write_bytes(token) + + +def load_config(*, data_dir: Path | None = None) -> ConfigPushRequest | None: + """Load + decrypt the persisted pushed config. + + Returns None (never raises) when the cache is absent, corrupt, or was written + under a different key, so a bad cache can never crash boot. + """ + data_dir = data_dir or _default_data_dir() + cache_path = data_dir / _CACHE_FILENAME + if not cache_path.exists(): + return None + try: + token = cache_path.read_bytes() + payload = _fernet(data_dir).decrypt(token) + return ConfigPushRequest.model_validate_json(payload) + except (InvalidToken, ValueError, OSError) as exc: + logger.warning("Ignoring unreadable AI config cache at %s: %s", cache_path, exc) + return None diff --git a/engine/src/stirling/config/settings.py b/engine/src/stirling/config/settings.py index 1293cc371f..add6c98972 100644 --- a/engine/src/stirling/config/settings.py +++ b/engine/src/stirling/config/settings.py @@ -25,6 +25,12 @@ class AppSettings(BaseSettings): smart_model_name: str = Field(validation_alias="STIRLING_SMART_MODEL") fast_model_name: str = Field(validation_alias="STIRLING_FAST_MODEL") + # Provider backing the active chat models. Empty for env/native providers; set to + # 'ollama'/'custom' by a config push. Agents that combine structured output with + # tools read this to pick a tool-compatible output strategy, because OpenAI-compatible + # local models (Ollama) block tool calls when a native json-schema response format is + # set, so their structured result must be delivered via a tool call instead. + chat_provider: str = Field(default="") smart_model_max_tokens: int = Field(validation_alias="STIRLING_SMART_MODEL_MAX_TOKENS") fast_model_max_tokens: int = Field(validation_alias="STIRLING_FAST_MODEL_MAX_TOKENS") # Process-wide ceiling on concurrent model API calls, shared by both model @@ -123,6 +129,11 @@ class AppSettings(BaseSettings): engine_shared_secret: str = Field(default="", validation_alias="STIRLING_ENGINE_SHARED_SECRET") engine_require_auth: bool = Field(default=False, validation_alias="STIRLING_ENGINE_REQUIRE_AUTH") + # When true, the Java processor may push admin-configured AI settings (model, + # credentials, limits) to POST /api/v1/config at startup. Turn this off in + # environment-driven deployments so the environment is the single source of truth. + allow_config_push: bool = Field(default=True, validation_alias="STIRLING_ALLOW_CONFIG_PUSH") + def _configure_logging(level_name: str, log_file: str, http_debug: bool) -> None: """Configure the ``stirling`` logger hierarchy.""" diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 513906d610..caf41effa0 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -29,6 +29,13 @@ from .common import ( format_conversation_history, format_file_names, ) +from .config import ( + ConfigApplyResponse, + ConfigLimitsSection, + ConfigModelsSection, + ConfigPushRequest, + ConfigRagSection, +) from .contradiction import ( Claim, Contradiction, @@ -139,6 +146,11 @@ __all__ = [ "Claim", "CommentSpec", "CompletedExecutionAction", + "ConfigApplyResponse", + "ConfigLimitsSection", + "ConfigModelsSection", + "ConfigPushRequest", + "ConfigRagSection", "Contradiction", "ContradictionReport", "ContradictionSeverity", diff --git a/engine/src/stirling/contracts/config.py b/engine/src/stirling/contracts/config.py new file mode 100644 index 0000000000..572cadba0c --- /dev/null +++ b/engine/src/stirling/contracts/config.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from pydantic import ConfigDict, Field + +from stirling.models import ApiModel + +# Tolerate unknown fields so a newer processor pushing a field this engine version +# doesn't know is ignored rather than rejecting the whole push. Overrides only the +# ``extra`` policy from ApiModel; the camelCase alias generator is inherited. +_TOLERANT = ConfigDict(extra="ignore") + + +class ConfigModelsSection(ApiModel): + """Model provider + credentials pushed by the Java processor. + + Field names on the wire are camelCase (``smartModel`` etc.) via the + :class:`ApiModel` alias generator. Empty ``provider``/``apiKey``/``baseUrl`` + or empty model names mean "keep the engine's env-configured value". + """ + + model_config = _TOLERANT + + provider: str = "" + smart_model: str = "" + fast_model: str = "" + smart_max_tokens: int | None = None + fast_max_tokens: int | None = None + api_key: str = "" + base_url: str = "" + + +class ConfigRagSection(ApiModel): + model_config = _TOLERANT + + embedding_provider: str = "" + embedding_model: str = "" + embedding_api_key: str = "" + # OpenAI-compatible endpoint URL for ollama/custom embedding providers. + # Empty keeps the engine's env value, same convention as the other fields. + embedding_base_url: str = "" + top_k: int | None = None + max_searches: int | None = None + + +class ConfigLimitsSection(ApiModel): + model_config = _TOLERANT + + max_pages: int | None = None + max_characters: int | None = None + model_max_concurrency: int | None = None + + +class ConfigPushRequest(ApiModel): + """Admin-configured AI settings pushed at processor startup.""" + + model_config = _TOLERANT + + models: ConfigModelsSection = Field(default_factory=ConfigModelsSection) + rag: ConfigRagSection = Field(default_factory=ConfigRagSection) + limits: ConfigLimitsSection = Field(default_factory=ConfigLimitsSection) + + +class ConfigApplyResponse(ApiModel): + """Summary of the effective config after a push. Never echoes credentials.""" + + status: str + provider: str + smart_model: str + fast_model: str + smart_max_tokens: int + fast_max_tokens: int + rag_embedding_model: str + rag_top_k: int + rag_max_searches: int + max_pages: int + max_characters: int + model_max_concurrency: int + notes: list[str] = Field(default_factory=list) diff --git a/engine/src/stirling/documents/embedder.py b/engine/src/stirling/documents/embedder.py index 13e9c7018d..9bd287a8d2 100644 --- a/engine/src/stirling/documents/embedder.py +++ b/engine/src/stirling/documents/embedder.py @@ -1,6 +1,8 @@ from __future__ import annotations from pydantic_ai import Embedder +from pydantic_ai.embeddings.openai import OpenAIEmbeddingModel +from pydantic_ai.providers.openai import OpenAIProvider from stirling.documents.chunker import chunk_text from stirling.documents.store import Document @@ -12,6 +14,37 @@ from stirling.documents.store import Document DEFAULT_EMBED_BATCH_SIZE = 256 +def _build_embedder( + model_name: str, + *, + provider: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> Embedder: + """Construct an :class:`Embedder` for ``model_name``. + + With no explicit ``provider``/``api_key``/``base_url`` this keeps the original + behaviour: ``model_name`` is the ``provider:model`` string form (e.g. + "voyageai:voyage-4") and credentials come from the provider's native env var. + + With an explicit provider (config-push path) ``model_name`` is the bare model + without a prefix. ``voyageai``/``openai`` compose the same string form; the + OpenAI-compatible ``ollama``/``custom`` build an OpenAI embedding model against + ``base_url`` (Ollama ignores the key but the SDK still needs a non-empty one). + """ + if not provider and not api_key and not base_url: + return Embedder(model_name) + + provider_name = (provider or "").lower() + key = api_key or None + if provider_name in ("voyageai", "openai"): + return Embedder(f"{provider_name}:{model_name}") + if provider_name in ("ollama", "custom"): + openai_provider = OpenAIProvider(base_url=base_url or None, api_key=key or "ollama") + return Embedder(OpenAIEmbeddingModel(model_name, provider=openai_provider)) + raise ValueError(f"Unsupported embedding provider {provider!r}.") + + class EmbeddingService: """Wraps Pydantic AI's Embedder to provide document chunking and embedding.""" @@ -21,8 +54,12 @@ class EmbeddingService: chunk_size: int = 512, chunk_overlap: int = 64, embed_batch_size: int = DEFAULT_EMBED_BATCH_SIZE, + *, + provider: str | None = None, + api_key: str | None = None, + base_url: str | None = None, ) -> None: - self._embedder = Embedder(model_name) + self._embedder = _build_embedder(model_name, provider=provider, api_key=api_key, base_url=base_url) self._chunk_size = chunk_size self._chunk_overlap = chunk_overlap self._embed_batch_size = embed_batch_size diff --git a/engine/src/stirling/documents/service.py b/engine/src/stirling/documents/service.py index 262f5f7ff4..7f90ac12d1 100644 --- a/engine/src/stirling/documents/service.py +++ b/engine/src/stirling/documents/service.py @@ -43,6 +43,25 @@ class DocumentService: self._store = store self._default_top_k = default_top_k + @property + def default_top_k(self) -> int: + return self._default_top_k + + @default_top_k.setter + def default_top_k(self, value: int) -> None: + # Lets a config-push retune retrieval breadth without rebuilding the store. + self._default_top_k = value + + @property + def embedder(self) -> EmbeddingService: + return self._embedder + + @embedder.setter + def embedder(self, value: EmbeddingService) -> None: + # Lets a config-push swap the embedding model while keeping the live store + # (and its connection pool). Existing vectors need re-indexing afterwards. + self._embedder = value + async def ingest( self, collection: FileId, diff --git a/engine/src/stirling/services/runtime.py b/engine/src/stirling/services/runtime.py index 011eb8aaf2..f0f5d5724f 100644 --- a/engine/src/stirling/services/runtime.py +++ b/engine/src/stirling/services/runtime.py @@ -1,7 +1,9 @@ from __future__ import annotations import asyncio +import json import logging +import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager from dataclasses import dataclass @@ -12,8 +14,10 @@ from pydantic_ai import RunContext from pydantic_ai.messages import ModelMessage, ModelResponse from pydantic_ai.models import Model, ModelRequestParameters, StreamedResponse, infer_model from pydantic_ai.models.anthropic import AnthropicModel +from pydantic_ai.models.openai import OpenAIChatModel from pydantic_ai.models.wrapper import WrapperModel from pydantic_ai.providers.anthropic import AnthropicProvider +from pydantic_ai.providers.openai import OpenAIProvider from pydantic_ai.settings import ModelSettings from stirling.config import ENGINE_ROOT, AppSettings, DocumentsBackend @@ -49,6 +53,79 @@ def _build_anthropic_http_client() -> httpx.AsyncClient: ) +# Placeholder used when no Anthropic key is configured; lets the engine boot so a +# deployment that only uses another provider (e.g. Ollama via the admin UI / config +# push) isn't blocked. Anthropic calls then fail with a clear 401 instead. +_UNCONFIGURED_ANTHROPIC_KEY = "unconfigured" +_warned_missing_anthropic_key = False + + +def _anthropic_provider(explicit_key: str | None = None) -> AnthropicProvider: + """Build the Anthropic provider, tolerating a missing key at startup. + + pydantic-ai's ``AnthropicProvider`` raises at construction when no key is + available, which would crash the whole engine at boot even for deployments + that never call Anthropic (Ollama/OpenAI-configured, or awaiting a config + push that supplies credentials). We fall back to a placeholder key and warn + once so startup succeeds; the placeholder only surfaces as a clear + provider-side auth error if an Anthropic call is actually made. + """ + http_client = _build_anthropic_http_client() + key = explicit_key or os.environ.get("ANTHROPIC_API_KEY") + if key: + return AnthropicProvider(api_key=key, http_client=http_client) + global _warned_missing_anthropic_key + if not _warned_missing_anthropic_key: + _warned_missing_anthropic_key = True + logger.warning( + "ANTHROPIC_API_KEY is not set - the engine will start, but Anthropic model " + "calls will fail until a key is provided or a different provider is configured " + "(admin AI settings / config push)." + ) + return AnthropicProvider(api_key=_UNCONFIGURED_ANTHROPIC_KEY, http_client=http_client) + + +class _NullContentCoercingTransport(httpx.AsyncHTTPTransport): + """Coerce assistant ``content: null`` to ``""`` in outgoing OpenAI chat requests. + + OpenAI's schema uses ``content: null`` for assistant messages that carry only + tool calls, but Ollama's OpenAI-compatible endpoint rejects it ("invalid message + content type: "), which intermittently breaks multi-turn tool conversations + (RAG search + tool-delivered output). Rewriting null -> "" makes them acceptable + without changing semantics. Scoped to the ollama/custom clients only. + """ + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.headers.get("content-type", "").startswith("application/json") and request.content: + try: + body = json.loads(request.content) + except ValueError: + return await super().handle_async_request(request) + messages = body.get("messages") + if isinstance(messages, list): + changed = False + for message in messages: + if isinstance(message, dict) and message.get("content", "") is None: + message["content"] = "" + changed = True + if changed: + new_body = json.dumps(body).encode("utf-8") + headers = [(k, v) for k, v in request.headers.raw if k.lower() != b"content-length"] + request = httpx.Request( + method=request.method, + url=request.url, + headers=headers, + content=new_body, + extensions=request.extensions, + ) + return await super().handle_async_request(request) + + +def _openai_compat_http_client() -> httpx.AsyncClient: + """httpx client for Ollama/custom OpenAI-compatible endpoints (null-content fix).""" + return httpx.AsyncClient(transport=_NullContentCoercingTransport()) + + class ConcurrencyLimitedModel(WrapperModel): """Caps in-flight model API calls with a semaphore shared across the process.""" @@ -131,45 +208,97 @@ def _build_document_store(settings: AppSettings) -> DocumentStore: assert_never(settings.documents_backend) -def _build_documents(settings: AppSettings) -> DocumentService: - """Build the document service used by per-request RAG capabilities.""" - logger.info("Documents: embedding_model=%s", settings.rag_embedding_model) - embedder = EmbeddingService( - model_name=settings.rag_embedding_model, - chunk_size=settings.rag_chunk_size, - chunk_overlap=settings.rag_chunk_overlap, - ) +def _build_documents(settings: AppSettings, embedder: EmbeddingService | None = None) -> DocumentService: + """Build the document service used by per-request RAG capabilities. + + ``embedder`` lets a cache-restore boot inject an already-built embedder (e.g. + an OpenAI-compatible ollama/custom one whose base_url the plain + ``settings.rag_embedding_model`` string can't encode); None builds from settings. + """ + if embedder is None: + logger.info("Documents: embedding_model=%s", settings.rag_embedding_model) + embedder = EmbeddingService( + model_name=settings.rag_embedding_model, + chunk_size=settings.rag_chunk_size, + chunk_overlap=settings.rag_chunk_overlap, + ) store = _build_document_store(settings) return DocumentService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k) -def build_runtime(settings: AppSettings) -> AppRuntime: - fast_model = _build_model(settings.fast_model_name) - smart_model = _build_model(settings.smart_model_name) - validate_structured_output_support(fast_model, settings.fast_model_name) - validate_structured_output_support(smart_model, settings.smart_model_name) +def build_runtime( + settings: AppSettings, + *, + documents: DocumentService | None = None, + fast_model: Model | None = None, + smart_model: Model | None = None, + embedder: EmbeddingService | None = None, +) -> AppRuntime: + """Assemble the shared runtime. + + ``documents``/``fast_model``/``smart_model`` let a config-push rebuild reuse + the live vector-store connection pool and inject already-built (and already + validated) models instead of reconstructing them from the model-name strings. + ``embedder`` injects a pre-built embedder when building a fresh document store + (cache-restore boot); it is ignored when ``documents`` is supplied. All None + (the lifespan path) preserves the original startup behaviour. + """ + fast = fast_model if fast_model is not None else _build_model(settings.fast_model_name) + smart = smart_model if smart_model is not None else _build_model(settings.smart_model_name) + validate_structured_output_support(fast, settings.fast_model_name) + validate_structured_output_support(smart, settings.smart_model_name) # One semaphore across both tiers: the cap protects the provider account # and process resources, which the tiers share. model_semaphore = asyncio.Semaphore(settings.model_max_concurrency) return AppRuntime( settings=settings, - fast_model=ConcurrencyLimitedModel(fast_model, model_semaphore), - smart_model=ConcurrencyLimitedModel(smart_model, model_semaphore), - documents=_build_documents(settings), + fast_model=ConcurrencyLimitedModel(fast, model_semaphore), + smart_model=ConcurrencyLimitedModel(smart, model_semaphore), + documents=documents if documents is not None else _build_documents(settings, embedder), ) -def _build_model(model_name: str) -> Model: - """Construct a model, injecting our keepalive-free httpx client for - Anthropic models so workers don't pick up stale pooled connections. +def _build_model( + model_name: str, + *, + provider: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> Model: + """Construct a model for ``model_name``. - Other providers fall back to ``infer_model`` defaults; the stale-pool - issue is specific to the Cloudflare-fronted Anthropic API in our - observations and the fix doesn't necessarily apply elsewhere. + With no explicit ``provider``/``api_key``/``base_url`` this keeps the original + env-driven behaviour: Anthropic-prefixed names get our keepalive-free httpx + client (so workers don't pick up stale pooled connections) and everything + else falls back to ``infer_model``. + + When the caller passes an explicit provider (config-push path), ``model_name`` + is the bare model without a ``provider:`` prefix and credentials come from the + call. An empty ``api_key``/``base_url`` is treated as unset so the provider + reads its native environment variable (e.g. ANTHROPIC_API_KEY / OPENAI_API_KEY). """ - if model_name.startswith("anthropic:"): - bare_name = model_name.removeprefix("anthropic:") - provider = AnthropicProvider(http_client=_build_anthropic_http_client()) - return AnthropicModel(bare_name, provider=provider) - return infer_model(model_name) + if not provider and not api_key and not base_url: + if model_name.startswith("anthropic:"): + bare_name = model_name.removeprefix("anthropic:") + return AnthropicModel(bare_name, provider=_anthropic_provider()) + return infer_model(model_name) + + provider_name = (provider or "").lower() + key = api_key or None + if provider_name == "anthropic": + return AnthropicModel(model_name, provider=_anthropic_provider(key)) + if provider_name == "openai": + openai_provider = OpenAIProvider(api_key=key) if key else OpenAIProvider() + return OpenAIChatModel(model_name, provider=openai_provider) + if provider_name in ("ollama", "custom"): + # OpenAI-compatible endpoint. Ollama ignores the key but the SDK still + # requires a non-empty one, so default to a harmless placeholder. The custom + # http client coerces null assistant content so multi-turn tool calls work. + openai_provider = OpenAIProvider( + base_url=base_url or None, + api_key=key or "ollama", + http_client=_openai_compat_http_client(), + ) + return OpenAIChatModel(model_name, provider=openai_provider) + raise ValueError(f"Unsupported model provider {provider!r}.") diff --git a/engine/tests/test_config_cache.py b/engine/tests/test_config_cache.py new file mode 100644 index 0000000000..235a45c15f --- /dev/null +++ b/engine/tests/test_config_cache.py @@ -0,0 +1,73 @@ +"""Tests for the encrypted config cache (stirling.config.config_cache).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stirling.config import config_cache +from stirling.contracts import ConfigPushRequest + + +def _sample() -> ConfigPushRequest: + return ConfigPushRequest.model_validate( + { + "models": { + "provider": "anthropic", + "smartModel": "claude-haiku-4-5", + "fastModel": "claude-haiku-4-5", + "smartMaxTokens": 8192, + "fastMaxTokens": 2048, + "apiKey": "secret-key-value", + "baseUrl": "", + }, + "rag": { + "embeddingProvider": "voyageai", + "embeddingModel": "voyage-4", + "embeddingApiKey": "embed-secret", + "embeddingBaseUrl": "", + "topK": 20, + "maxSearches": 5, + }, + "limits": {"maxPages": 200, "maxCharacters": 200000, "modelMaxConcurrency": 32}, + } + ) + + +def test_roundtrip_with_shared_secret(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(config_cache, "_shared_secret", lambda: "the-shared-secret") + req = _sample() + config_cache.save_config(req, data_dir=tmp_path) + # HKDF-from-secret path: no keyfile is written. + assert not (tmp_path / "ai_config_cache.key").exists() + assert config_cache.load_config(data_dir=tmp_path) == req + # Secrets are encrypted at rest. + assert b"secret-key-value" not in (tmp_path / "ai_config_cache.enc").read_bytes() + + +def test_roundtrip_with_keyfile_when_no_secret(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(config_cache, "_shared_secret", lambda: "") + req = _sample() + config_cache.save_config(req, data_dir=tmp_path) + # No shared secret: a random keyfile is generated and reused for decrypt. + assert (tmp_path / "ai_config_cache.key").exists() + assert config_cache.load_config(data_dir=tmp_path) == req + + +def test_load_missing_returns_none(tmp_path: Path) -> None: + assert config_cache.load_config(data_dir=tmp_path) is None + + +def test_load_corrupt_returns_none(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(config_cache, "_shared_secret", lambda: "a-secret") + (tmp_path / "ai_config_cache.enc").write_bytes(b"not-a-valid-fernet-token") + assert config_cache.load_config(data_dir=tmp_path) is None + + +def test_wrong_key_returns_none(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(config_cache, "_shared_secret", lambda: "secret-a") + config_cache.save_config(_sample(), data_dir=tmp_path) + # A different secret derives a different key -> decrypt fails, returns None. + monkeypatch.setattr(config_cache, "_shared_secret", lambda: "secret-b") + assert config_cache.load_config(data_dir=tmp_path) is None diff --git a/engine/tests/test_config_routes.py b/engine/tests/test_config_routes.py new file mode 100644 index 0000000000..615279257f --- /dev/null +++ b/engine/tests/test_config_routes.py @@ -0,0 +1,249 @@ +"""Tests for the config-push endpoint (POST /api/v1/config). + +These drive the real lifespan (via ``with TestClient(app)``) so ``app.state`` is +populated, then exercise the gate + model-swap behaviour against the in-memory +"test" runtime built by :func:`build_app_settings`. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import patch + +import pytest +from conftest import build_app_settings +from fastapi.testclient import TestClient + +from stirling.api import app +from stirling.config import AppSettings, config_cache, load_settings +from stirling.contracts import ConfigPushRequest + + +@pytest.fixture(autouse=True) +def _isolate_config_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Point the encrypted config cache at a per-test tmp dir so persistence never + touches the real engine/data dir or leaks between tests.""" + monkeypatch.setattr(config_cache, "_default_data_dir", lambda: tmp_path) + + +@contextmanager +def _client( + settings_factory: Callable[[], AppSettings], + *, + client_addr: tuple[str, int] = ("127.0.0.1", 12345), +) -> Iterator[TestClient]: + """Enter a TestClient whose lifespan builds app.state from ``settings_factory``. + + Defaults to a loopback client address so the config endpoint's local-caller gate + (which blocks remote unauthenticated pushes) allows the request through. + """ + previous = app.dependency_overrides.get(load_settings) + app.dependency_overrides[load_settings] = settings_factory + try: + with TestClient(app, client=client_addr) as client: + yield client + finally: + if previous is None: + app.dependency_overrides.pop(load_settings, None) + else: + app.dependency_overrides[load_settings] = previous + + +def _anthropic_push() -> dict[str, object]: + return { + "models": { + "provider": "anthropic", + "smartModel": "claude-haiku-4-5", + "fastModel": "claude-haiku-4-5", + "smartMaxTokens": 4096, + "fastMaxTokens": 1024, + "apiKey": "test-key-not-a-real-secret", + "baseUrl": "", + }, + "rag": { + "embeddingProvider": "", + "embeddingModel": "", + "embeddingApiKey": "", + "embeddingBaseUrl": "", + "topK": 7, + "maxSearches": 3, + }, + "limits": {"maxPages": 50, "maxCharacters": 12345, "modelMaxConcurrency": 8}, + } + + +def test_config_push_forbidden_when_disabled() -> None: + def factory() -> AppSettings: + return build_app_settings().model_copy(update={"allow_config_push": False}) + + with _client(factory) as client: + response = client.post("/api/v1/config", json=_anthropic_push()) + assert response.status_code == 403 + + +def test_config_push_from_non_local_caller_without_secret_returns_403() -> None: + """Secure-by-default: with no shared secret, a remote caller cannot push a config.""" + with _client(build_app_settings, client_addr=("203.0.113.9", 4444)) as client: + response = client.post("/api/v1/config", json=_anthropic_push()) + assert response.status_code == 403 + assert "STIRLING_ENGINE_SHARED_SECRET" in response.json()["detail"] + + +def test_config_push_applies_model_and_limits() -> None: + with _client(build_app_settings) as client: + response = client.post("/api/v1/config", json=_anthropic_push()) + assert response.status_code == 200 + body = response.json() + # Wire summary is camelCase and never echoes the api key. + assert body["smartModel"] == "claude-haiku-4-5" + assert body["fastModel"] == "claude-haiku-4-5" + assert body["smartMaxTokens"] == 4096 + assert body["ragTopK"] == 7 + assert body["maxPages"] == 50 + assert body["modelMaxConcurrency"] == 8 + assert "test-key-not-a-real-secret" not in response.text + # State was swapped: the running settings now reflect the push. + assert app.state.settings.smart_model_name == "claude-haiku-4-5" + assert app.state.settings.max_pages == 50 + assert app.state.runtime.documents.default_top_k == 7 + + +def test_config_push_unsupported_provider_returns_400_without_swap() -> None: + with _client(build_app_settings) as client: + before = app.state.runtime + payload = _anthropic_push() + payload["models"]["provider"] = "nonsense-provider" # type: ignore[index] + response = client.post("/api/v1/config", json=payload) + assert response.status_code == 400 + # Running runtime is untouched when the push is rejected. + assert app.state.runtime is before + assert app.state.settings.smart_model_name == "test" + + +def test_config_push_unsupported_model_returns_400() -> None: + """A model that fails structured-output validation is rejected with 400.""" + with _client(build_app_settings) as client: + before = app.state.runtime + with patch( + "stirling.api.routes.config.validate_structured_output_support", + side_effect=ValueError("Unsupported model foo. This model does not support structured outputs."), + ): + response = client.post("/api/v1/config", json=_anthropic_push()) + assert response.status_code == 400 + assert "does not support structured outputs" in response.json()["detail"] + assert app.state.runtime is before + + +def test_config_push_ollama_embedding_rebuilds_embedder() -> None: + """A pushed ollama/custom embedding provider builds an OpenAI-compatible + embedder and swaps it onto the reused DocumentService without tearing down + the store; the response carries a re-index note.""" + with _client(build_app_settings) as client: + store_before = app.state.runtime.documents + embedder_before = app.state.runtime.documents.embedder + payload = _anthropic_push() + payload["rag"] = { + "embeddingProvider": "ollama", + "embeddingModel": "nomic-embed-text", + "embeddingApiKey": "", + "embeddingBaseUrl": "http://localhost:11434/v1", + "topK": 9, + "maxSearches": 4, + } + response = client.post("/api/v1/config", json=payload) + assert response.status_code == 200 + body = response.json() + assert body["ragEmbeddingModel"] == "ollama:nomic-embed-text" + assert any("re-index" in note for note in body["notes"]) + docs = app.state.runtime.documents + # Same store object reused (connection pool intact), embedder swapped. + assert docs is store_before + assert docs.embedder is not embedder_before + assert docs.default_top_k == 9 + + +def test_config_push_unsupported_embedding_provider_returns_400() -> None: + with _client(build_app_settings) as client: + embedder_before = app.state.runtime.documents.embedder + payload = _anthropic_push() + payload["rag"] = { + "embeddingProvider": "totally-bogus", + "embeddingModel": "x", + "embeddingApiKey": "", + "embeddingBaseUrl": "", + "topK": None, + "maxSearches": None, + } + response = client.post("/api/v1/config", json=payload) + assert response.status_code == 400 + # Embedder untouched when the push is rejected. + assert app.state.runtime.documents.embedder is embedder_before + + +def test_config_push_empty_models_keep_env_value() -> None: + """An empty models block keeps the engine's current (env) models but still + applies pushed limits.""" + with _client(build_app_settings) as client: + payload = _anthropic_push() + payload["models"] = { + "provider": "", + "smartModel": "", + "fastModel": "", + "smartMaxTokens": None, + "fastMaxTokens": None, + "apiKey": "", + "baseUrl": "", + } + response = client.post("/api/v1/config", json=payload) + assert response.status_code == 200 + # Env "test" model preserved for both tiers; limits still applied. + assert app.state.settings.smart_model_name == "test" + assert app.state.settings.fast_model_name == "test" + assert app.state.settings.max_pages == 50 + + +def test_config_push_ignores_unknown_fields() -> None: + """A newer processor pushing fields this engine doesn't know must not 422; + unknown fields are ignored and the rest of the push still applies.""" + with _client(build_app_settings) as client: + payload = _anthropic_push() + payload["futureTopLevelField"] = {"anything": 1} + payload["models"]["experimentalFlag"] = True # type: ignore[index] + response = client.post("/api/v1/config", json=payload) + assert response.status_code == 200 + assert app.state.settings.smart_model_name == "claude-haiku-4-5" + assert app.state.settings.max_pages == 50 + + +def test_boot_restores_cached_config() -> None: + """A persisted config is decrypted and applied on boot, overriding env.""" + config_cache.save_config(ConfigPushRequest.model_validate(_anthropic_push())) + with _client(build_app_settings): + # Env model is "test"; the cache pushed claude-haiku-4-5 + limits. + assert app.state.settings.smart_model_name == "claude-haiku-4-5" + assert app.state.settings.fast_model_name == "claude-haiku-4-5" + assert app.state.settings.max_pages == 50 + assert app.state.settings.smart_model_max_tokens == 4096 + assert app.state.runtime.documents.default_top_k == 7 + + +def test_boot_ignores_cache_when_push_disabled() -> None: + """With allow_config_push false, env wins and the cache is ignored.""" + config_cache.save_config(ConfigPushRequest.model_validate(_anthropic_push())) + + def factory() -> AppSettings: + return build_app_settings().model_copy(update={"allow_config_push": False}) + + with _client(factory): + assert app.state.settings.smart_model_name == "test" + assert app.state.settings.max_pages == 200 + + +def test_boot_proceeds_on_corrupt_cache(tmp_path: Path) -> None: + """A corrupt cache file is ignored and boot falls back to env, never crashing.""" + (tmp_path / "ai_config_cache.enc").write_bytes(b"not-a-valid-fernet-token") + with _client(build_app_settings): + assert app.state.settings.smart_model_name == "test" + assert app.state.settings.max_pages == 200 diff --git a/engine/uv.lock b/engine/uv.lock index bac51cb3c9..a1917ec708 100644 --- a/engine/uv.lock +++ b/engine/uv.lock @@ -603,6 +603,7 @@ name = "engine" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "cryptography" }, { name = "fastapi" }, { name = "jinja2" }, { name = "opentelemetry-sdk" }, @@ -630,6 +631,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cryptography", specifier = ">=44.0.0" }, { name = "fastapi", specifier = ">=0.116.0" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "opentelemetry-sdk", specifier = ">=1.39.0" }, diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index c182ce2b89..84b79af607 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -692,6 +692,160 @@ manualLinks = "Manual downloads: click the links and place the files into the te noLanguages = "No tessdata languages found in the configured directory." permissionNotice = "The tessdata path is not writable. Downloads will be opened in the browser; please save the .traineddata files manually into the tessdata folder." +# AI engine admin settings (AI nav group) +[admin.settings.ai.documents] +description = "Configure the embedding model and retrieval settings used to answer questions over documents. Applied to the AI engine when saved." +title = "Documents & RAG" + +[admin.settings.ai.documents.embeddingApiKey] +description = "Leave blank to use the engine's own environment credential. Applies to self-hosted single-engine deployments." +generic = "Embedding API key" +openai = "OpenAI API key" +voyageai = "VoyageAI API key" + +[admin.settings.ai.documents.embeddingBaseUrl] +description = "Base URL of the OpenAI-compatible / Ollama embeddings endpoint, e.g. http://ollama:11434/v1. Must point at a trusted internal endpoint (SSRF-sensitive)." +label = "Embedding base URL" + +[admin.settings.ai.documents.embeddingModel] +description = "Embedding model name. Free text; suggestions are hints only." +label = "Embedding model" + +[admin.settings.ai.documents.embeddingProvider] +description = "Provider used to turn document text into vector embeddings." +label = "Embedding provider" + +[admin.settings.ai.documents.maxSearches] +description = "Maximum number of retrieval searches the agent may run per request." +label = "Max searches" + +[admin.settings.ai.documents.reindexNote] +body = "Changing the embedding model invalidates previously-embedded documents. A full engine restart and re-index of existing documents is required for search to work correctly." +title = "Re-index required" + +[admin.settings.ai.documents.topK] +description = "Number of most-relevant chunks retrieved per search." +label = "Top K" + +[admin.settings.ai.general] +description = "Connect Stirling to the Python AI engine and choose which AI capabilities are exposed. Changes apply on restart." +title = "AI Engine" + +[admin.settings.ai.general.capabilities] +description = "Turn individual AI features on or off. Disabled features are hidden in the app." +title = "Capabilities" + +[admin.settings.ai.general.enabled] +description = "Master switch. When off, no AI tools, agents, or engine calls are available." +label = "Enable AI" + +[admin.settings.ai.general.features.chat] +description = "Conversational assistant for working with PDFs." +label = "Chat assistant" + +[admin.settings.ai.general.features.classify] +description = "Automatically categorise documents by type or content." +label = "Document classification" + +[admin.settings.ai.general.features.createPdf] +description = "Generate a new PDF (e.g. from HTML) via an AI agent." +label = "Create PDF from prompt" + +[admin.settings.ai.general.features.documentQuestions] +description = "Ask questions and get answers grounded in an uploaded document." +label = "Document questions" + +[admin.settings.ai.general.features.mathAuditor] +description = "Review documents for mathematical and numerical errors." +label = "Math auditor" + +[admin.settings.ai.general.features.pdfComment] +description = "Add AI-authored review comments and annotations to a PDF." +label = "PDF comment agent" + +[admin.settings.ai.general.longRunningTimeoutSeconds] +description = "Timeout for heavier agent operations such as document generation." +label = "Long-running timeout (seconds)" + +[admin.settings.ai.general.note] +body = "The AI engine runs as a separate service. Its shared secret" +body2 = "is set via a container environment variable. Provider API keys can be entered on these pages or supplied as engine environment variables; keys entered here are pushed to the engine when saved." +title = "About the AI engine" + +[admin.settings.ai.general.streamTimeoutSeconds] +description = "Timeout for streamed (token-by-token) chat responses." +label = "Stream timeout (seconds)" + +[admin.settings.ai.general.test] +button = "Test connection" +failBody = "The AI engine did not respond. Check the URL, that the engine container is running, and that AI is enabled (a restart is needed after enabling)." +failTitle = "AI engine unreachable" +okBody = "The AI engine responded to a health check." +okTitle = "AI engine reachable" + +[admin.settings.ai.general.timeoutSeconds] +description = "Timeout for standard AI requests to the engine." +label = "Request timeout (seconds)" + +[admin.settings.ai.general.url] +description = "Internal URL of the Python AI engine, e.g. http://stirling-pdf-engine:5001." +label = "AI engine URL" + +[admin.settings.ai.limits] +description = "Guardrails for how much work AI requests may do and how many run concurrently. Applied to the AI engine when saved." +title = "Limits & Performance" + +[admin.settings.ai.limits.maxCharacters] +description = "Guardrail: reject AI requests whose extracted text exceeds this length." +label = "Max characters per request" + +[admin.settings.ai.limits.maxPages] +description = "Guardrail: reject AI requests over this many PDF pages." +label = "Max pages per request" + +[admin.settings.ai.limits.modelMaxConcurrency] +description = "Maximum simultaneous in-flight model calls across the whole engine." +label = "Model max concurrency" + +[admin.settings.ai.models] +description = "Choose the LLM provider and the smart/fast models the AI engine uses. Applied to the AI engine when saved." +title = "Models & Providers" + +[admin.settings.ai.models.apiKey] +anthropic = "Anthropic API key" +description = "Leave blank to use the engine's own environment credential. Applies to self-hosted single-engine deployments." +generic = "API key" +openai = "OpenAI API key" + +[admin.settings.ai.models.baseUrl] +description = "Base URL of the OpenAI-compatible / Ollama endpoint, e.g. http://ollama:11434/v1." +label = "Provider base URL" +warning = "The base URL must point at a trusted internal endpoint. The engine will make server-side requests to it, so an untrusted value is SSRF-sensitive." + +[admin.settings.ai.models.fastMaxTokens] +description = "Maximum output tokens for the fast model." +label = "Fast model max tokens" + +[admin.settings.ai.models.fastModel] +description = "Cheaper, faster model for lightweight tasks. Free text; suggestions are hints only." +label = "Fast model" + +[admin.settings.ai.models.provider] +description = "Which LLM provider the engine talks to." +label = "Provider" + +[admin.settings.ai.models.smartMaxTokens] +description = "Maximum output tokens for the smart model." +label = "Smart model max tokens" + +[admin.settings.ai.models.smartModel] +description = "High-capability model for complex reasoning. Free text; suggestions are hints only." +label = "Smart model" + +[admin.settings.ai.saved] +body = "Changes are pushed to the AI engine automatically." +title = "AI settings saved" + [admin.settings.badge] clickToUpgrade = "Click to view plan details" @@ -8952,6 +9106,13 @@ title = "Signature Order" [settings] close = "Close" +[settings.ai] +documents = "Documents & RAG" +general = "General" +limits = "Limits & Performance" +models = "Models & Providers" +title = "AI" + [settings.configuration] advanced = "Advanced" database = "Database" diff --git a/frontend/editor/src/core/components/shared/config/types.ts b/frontend/editor/src/core/components/shared/config/types.ts index 5cbdd1b169..1182933d61 100644 --- a/frontend/editor/src/core/components/shared/config/types.ts +++ b/frontend/editor/src/core/components/shared/config/types.ts @@ -32,6 +32,10 @@ export const VALID_NAV_KEYS = [ "adminEndpoints", "adminStorageSharing", "adminMcp", + "adminAiGeneral", + "adminAiModels", + "adminAiDocuments", + "adminAiLimits", "help", "legal", "backendThirdPartyLicenses", diff --git a/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx b/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx index f9c19f8a7b..35f5eaea45 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx @@ -17,6 +17,10 @@ import AdminPlanSection from "@app/components/shared/config/configSections/Admin import AdminFeaturesSection from "@app/components/shared/config/configSections/AdminFeaturesSection"; import AdminEndpointsSection from "@app/components/shared/config/configSections/AdminEndpointsSection"; import AdminMcpSection from "@app/components/shared/config/configSections/AdminMcpSection"; +import AdminAiGeneralSection from "@app/components/shared/config/configSections/AdminAiGeneralSection"; +import AdminAiModelsSection from "@app/components/shared/config/configSections/AdminAiModelsSection"; +import AdminAiDocumentsSection from "@app/components/shared/config/configSections/AdminAiDocumentsSection"; +import AdminAiLimitsSection from "@app/components/shared/config/configSections/AdminAiLimitsSection"; import AdminAuditSection from "@app/components/shared/config/configSections/AdminAuditSection"; import AdminUsageSection from "@app/components/shared/config/configSections/AdminUsageSection"; import AdminStorageSharingSection from "@app/components/shared/config/configSections/AdminStorageSharingSection"; @@ -163,6 +167,45 @@ export const useConfigNavSections = ( ], }); + // AI + sections.push({ + title: t("settings.ai.title", "AI"), + items: [ + { + key: "adminAiGeneral", + label: t("settings.ai.general", "General"), + icon: "smart-toy-rounded", + component: , + disabled: requiresLogin, + disabledTooltip: requiresLogin ? enableLoginTooltip : undefined, + }, + { + key: "adminAiModels", + label: t("settings.ai.models", "Models & Providers"), + icon: "psychology-rounded", + component: , + disabled: requiresLogin, + disabledTooltip: requiresLogin ? enableLoginTooltip : undefined, + }, + { + key: "adminAiDocuments", + label: t("settings.ai.documents", "Documents & RAG"), + icon: "description-rounded", + component: , + disabled: requiresLogin, + disabledTooltip: requiresLogin ? enableLoginTooltip : undefined, + }, + { + key: "adminAiLimits", + label: t("settings.ai.limits", "Limits & Performance"), + icon: "speed-rounded", + component: , + disabled: requiresLogin, + disabledTooltip: requiresLogin ? enableLoginTooltip : undefined, + }, + ], + }); + // Security & Authentication sections.push({ title: t("settings.securityAuth.title", "Security & Authentication"), diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAiDocumentsSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAiDocumentsSection.tsx new file mode 100644 index 0000000000..3f0bb66224 --- /dev/null +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAiDocumentsSection.tsx @@ -0,0 +1,343 @@ +import { useCallback, useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { + NumberInput, + PasswordInput, + Autocomplete, + Select, + TextInput, + Stack, + Paper, + Text, + Loader, + Group, + Alert, +} from "@mantine/core"; +import { alert } from "@app/components/toast"; +import LocalIcon from "@app/components/shared/LocalIcon"; +import { useAdminSettings } from "@app/hooks/useAdminSettings"; +import { useSettingsDirty } from "@app/hooks/useSettingsDirty"; +import PendingBadge from "@app/components/shared/config/PendingBadge"; +import { SettingsStickyFooter } from "@app/components/shared/config/SettingsStickyFooter"; +import apiClient from "@app/services/apiClient"; +import { useLoginRequired } from "@app/hooks/useLoginRequired"; +import { Z_INDEX_OVER_CONFIG_MODAL } from "@app/styles/zIndex"; +import { + AiEngineSettingsData, + AiEngineRag, + AiEngineApiResponse, + EMBEDDING_MODEL_SUGGESTIONS, +} from "@app/components/shared/config/configSections/aiEngineSettings"; + +export default function AdminAiDocumentsSection() { + const { t } = useTranslation(); + const { loginEnabled } = useLoginRequired(); + + // Track edits to the masked secret so we never send "********" back. + const [embeddingApiKeyDirty, setEmbeddingApiKeyDirty] = useState(false); + + const { + settings, + setSettings, + loading, + saving, + fetchSettings, + saveSettings, + isFieldPending, + } = useAdminSettings({ + sectionName: "aiEngine", + fetchTransformer: async (): Promise< + AiEngineSettingsData & { _pending?: Partial } + > => { + const response = await apiClient.get( + "/api/v1/admin/settings/section/aiEngine", + ); + return response.data || {}; + }, + // Save ONLY this page's keys as dot-notation so sibling AI keys are preserved. + saveTransformer: (s: AiEngineSettingsData) => { + const deltaSettings: Record = { + "aiEngine.rag.embeddingProvider": + s.rag?.embeddingProvider ?? "voyageai", + "aiEngine.rag.embeddingModel": s.rag?.embeddingModel ?? "", + "aiEngine.rag.embeddingBaseUrl": s.rag?.embeddingBaseUrl ?? "", + "aiEngine.rag.topK": s.rag?.topK ?? 0, + "aiEngine.rag.maxSearches": s.rag?.maxSearches ?? 0, + }; + if (embeddingApiKeyDirty) { + deltaSettings["aiEngine.rag.embeddingApiKey"] = + s.rag?.embeddingApiKey ?? ""; + } + return { sectionData: {}, deltaSettings }; + }, + }); + + useEffect(() => { + fetchSettings(); + }, []); + + const { isDirty, resetToSnapshot, markSaved } = useSettingsDirty( + settings, + loading, + ); + + const handleSave = async () => { + try { + await saveSettings(); + setEmbeddingApiKeyDirty(false); + markSaved(); + // Engine-facing values are pushed to the AI engine live on save; no restart needed. + alert({ + alertType: "success", + title: t("admin.settings.ai.saved.title", "AI settings saved"), + body: t( + "admin.settings.ai.saved.body", + "Changes are pushed to the AI engine automatically.", + ), + }); + } catch (_error) { + alert({ + alertType: "error", + title: t("admin.error", "Error"), + body: t("admin.settings.saveError", "Failed to save settings"), + }); + } + }; + + const handleDiscard = useCallback(() => { + setEmbeddingApiKeyDirty(false); + setSettings(resetToSnapshot()); + }, [resetToSnapshot, setSettings]); + + const setRag = (patch: Partial) => + setSettings({ ...settings, rag: { ...(settings.rag || {}), ...patch } }); + + if (loading) { + return ( + + + + ); + } + + const embeddingProvider = settings.rag?.embeddingProvider || "voyageai"; + const embeddingSuggestions = + EMBEDDING_MODEL_SUGGESTIONS[embeddingProvider] || []; + const showEmbeddingBaseUrl = + embeddingProvider === "ollama" || embeddingProvider === "custom"; + // Ollama's embeddings endpoint needs no API key, mirroring the Models page. + const showEmbeddingApiKey = embeddingProvider !== "ollama"; + const embeddingApiKeyLabel = + embeddingProvider === "voyageai" + ? t( + "admin.settings.ai.documents.embeddingApiKey.voyageai", + "VoyageAI API key", + ) + : embeddingProvider === "openai" + ? t( + "admin.settings.ai.documents.embeddingApiKey.openai", + "OpenAI API key", + ) + : t( + "admin.settings.ai.documents.embeddingApiKey.generic", + "Embedding API key", + ); + const embeddingApiKeyPlaceholder = + embeddingProvider === "voyageai" + ? "pa-..." + : embeddingProvider === "openai" + ? "sk-..." + : ""; + + return ( +
+ +
+ + {t("admin.settings.ai.documents.title", "Documents & RAG")} + + + {t( + "admin.settings.ai.documents.description", + "Configure the embedding model and retrieval settings used to answer questions over documents. Applied to the AI engine when saved.", + )} + +
+ + + + + + {t("admin.settings.ai.models.provider.label", "Provider")} + + + + } + description={t( + "admin.settings.ai.models.provider.description", + "Which LLM provider the engine talks to.", + )} + data={[ + { value: "anthropic", label: "Anthropic" }, + { value: "openai", label: "OpenAI" }, + { value: "ollama", label: "Ollama" }, + { value: "custom", label: "Custom (OpenAI-compatible)" }, + ]} + value={provider} + onChange={(v) => setModels({ provider: v || "anthropic" })} + allowDeselect={false} + comboboxProps={{ + withinPortal: true, + zIndex: Z_INDEX_OVER_CONFIG_MODAL, + }} + /> + + + + {t( + "admin.settings.ai.models.smartModel.label", + "Smart model", + )} + + + + } + description={t( + "admin.settings.ai.models.smartModel.description", + "High-capability model for complex reasoning. Free text; suggestions are hints only.", + )} + data={modelSuggestions} + value={settings.models?.smartModel || ""} + onChange={(value) => setModels({ smartModel: value })} + comboboxProps={{ + withinPortal: true, + zIndex: Z_INDEX_OVER_CONFIG_MODAL, + }} + /> + + + + {t( + "admin.settings.ai.models.fastModel.label", + "Fast model", + )} + + + + } + description={t( + "admin.settings.ai.models.fastModel.description", + "Cheaper, faster model for lightweight tasks. Free text; suggestions are hints only.", + )} + data={modelSuggestions} + value={settings.models?.fastModel || ""} + onChange={(value) => setModels({ fastModel: value })} + comboboxProps={{ + withinPortal: true, + zIndex: Z_INDEX_OVER_CONFIG_MODAL, + }} + /> + + + + {t( + "admin.settings.ai.models.smartMaxTokens.label", + "Smart model max tokens", + )} + + + + } + description={t( + "admin.settings.ai.models.smartMaxTokens.description", + "Maximum output tokens for the smart model.", + )} + value={settings.models?.smartMaxTokens ?? 0} + onChange={(value) => setModels({ smartMaxTokens: Number(value) })} + min={1} + /> + + + + {t( + "admin.settings.ai.models.fastMaxTokens.label", + "Fast model max tokens", + )} + + + + } + description={t( + "admin.settings.ai.models.fastMaxTokens.description", + "Maximum output tokens for the fast model.", + )} + value={settings.models?.fastMaxTokens ?? 0} + onChange={(value) => setModels({ fastMaxTokens: Number(value) })} + min={1} + /> + + {showApiKey && ( + + {apiKeyLabel} + + + } + description={t( + "admin.settings.ai.models.apiKey.description", + "Leave blank to use the engine's own environment credential. Applies to self-hosted single-engine deployments.", + )} + value={settings.models?.apiKey || ""} + onChange={(e) => { + setApiKeyDirty(true); + setModels({ apiKey: e.target.value }); + }} + placeholder={apiKeyPlaceholder} + /> + )} + + {showBaseUrl && ( + + + {t( + "admin.settings.ai.models.baseUrl.label", + "Provider base URL", + )} + + + + } + description={t( + "admin.settings.ai.models.baseUrl.description", + "Base URL of the OpenAI-compatible / Ollama endpoint, e.g. http://ollama:11434/v1.", + )} + value={settings.models?.baseUrl || ""} + onChange={(e) => setModels({ baseUrl: e.target.value })} + placeholder="http://ollama:11434/v1" + /> + )} + + {showBaseUrl && ( + + } + > + + {t( + "admin.settings.ai.models.baseUrl.warning", + "The base URL must point at a trusted internal endpoint. The engine will make server-side requests to it, so an untrusted value is SSRF-sensitive.", + )} + + + )} + + + + + +
+ ); +} diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/aiEngineSettings.ts b/frontend/editor/src/proprietary/components/shared/config/configSections/aiEngineSettings.ts new file mode 100644 index 0000000000..908e72cd87 --- /dev/null +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/aiEngineSettings.ts @@ -0,0 +1,75 @@ +// Shared types + constants for the aiEngine admin settings section. +// The four AI sub-pages fetch the same GET /section/aiEngine payload but each +// saves only its own dot-notation keys so sibling keys are preserved. + +/** Secret fields come back masked as this literal when a value is set. */ +export const MASKED_SECRET = "********"; + +export interface AiEngineModels { + provider?: string; + smartModel?: string; + fastModel?: string; + smartMaxTokens?: number; + fastMaxTokens?: number; + apiKey?: string; + baseUrl?: string; +} + +export interface AiEngineRag { + embeddingProvider?: string; + embeddingModel?: string; + embeddingApiKey?: string; + embeddingBaseUrl?: string; + topK?: number; + maxSearches?: number; +} + +export interface AiEngineLimits { + maxPages?: number; + maxCharacters?: number; + modelMaxConcurrency?: number; +} + +export interface AiEngineFeatures { + chat?: boolean; + documentQuestions?: boolean; + createPdf?: boolean; + mathAuditor?: boolean; + pdfComment?: boolean; + classify?: boolean; +} + +export interface AiEngineSettingsData { + enabled?: boolean; + url?: string; + timeoutSeconds?: number; + longRunningTimeoutSeconds?: number; + streamTimeoutSeconds?: number; + models?: AiEngineModels; + rag?: AiEngineRag; + limits?: AiEngineLimits; + features?: AiEngineFeatures; +} + +export interface ApiResponseWithPending { + _pending?: Partial; +} + +export type AiEngineApiResponse = AiEngineSettingsData & + ApiResponseWithPending; + +/** Free-text model suggestions per provider (hints only). */ +export const MODEL_SUGGESTIONS: Record = { + anthropic: ["claude-haiku-4-5", "claude-sonnet-4-5", "claude-opus-4-1"], + openai: ["gpt-4o", "gpt-4o-mini", "o3-mini"], + ollama: ["llama3.1", "qwen2.5", "mistral"], + custom: ["llama3.1", "qwen2.5", "mistral"], +}; + +/** Free-text embedding-model suggestions per embedding provider. */ +export const EMBEDDING_MODEL_SUGGESTIONS: Record = { + voyageai: ["voyage-4", "voyage-3.5"], + openai: ["text-embedding-3-small", "text-embedding-3-large"], + ollama: ["nomic-embed-text", "mxbai-embed-large", "bge-m3"], + custom: ["nomic-embed-text", "mxbai-embed-large", "bge-m3"], +};