Merge origin/main into AI-Form-Fill

Resolution favours main for all conflicts (new orchestrator shape with
artifacts/file_names/conversation_history/resume_with, WorkflowOutcome
enum, RAG, ledger agent, deleted engine/config/.env.example).

Re-wires our three form-fill agents (FormAnalyserAgent, FormFillerAgent,
DocumentExtractorAgent) into the new app layout:
- contracts/__init__.py: adds form-fill exports to __all__
- api/dependencies.py: adds get_form_analyser_agent / get_form_filler_agent / get_document_extractor_agent
- api/app.py: instantiates the three agents in lifespan and registers form_fill_router
- api/routes/__init__.py: exports form_fill_router
- tests/test_stirling_api.py: imports and registers form-fill stubs
- tests/test_stirling_contracts.py: adds back KnowledgeUpdateResponse discriminator test

Form fill remains accessible via its own endpoints (/api/v1/form/ai/analyse,
/fill-batch, /extract). Not wired as an orchestrator delegate — following
main's pattern where orchestrator only routes pdf_edit/pdf_question/user_spec
/math_auditor.

Follow-ups still needed:
- Thread conversation_history into form-fill agent prompts
- Align form-fill response outcomes with WorkflowOutcome enum
- Decide whether engine should return ToolOperationStep plans (main's
  new pattern per #6116) or keep returning fill values directly

All 127 engine tests pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

engine/src/stirling/api/app.py

@@ -4,6 +4,8 @@ from contextlib import asynccontextmanager
 from typing import Annotated
 
 from fastapi import Depends, FastAPI
+from pydantic_ai import Agent
+from pydantic_ai.models.instrumented import InstrumentationSettings
 
 from stirling.agents import (
     DocumentExtractorAgent,
@@ -15,17 +17,21 @@ from stirling.agents import (
     PdfQuestionAgent,
     UserSpecAgent,
 )
+from stirling.agents.ledger import MathAuditorAgent
+from stirling.api.middleware import UserIdMiddleware
 from stirling.api.routes import (
     agent_draft_router,
     execution_router,
     form_fill_router,
+    ledger_router,
     orchestrator_router,
     pdf_edit_router,
     pdf_question_router,
+    rag_router,
 )
 from stirling.config import AppSettings, load_settings
 from stirling.contracts import HealthResponse
-from stirling.services import build_runtime
+from stirling.services import build_runtime, setup_posthog_tracking
 
 
 def _load_startup_settings(fast_api: FastAPI) -> AppSettings:
@@ -46,18 +52,27 @@ async def lifespan(fast_api: FastAPI):
     fast_api.state.pdf_question_agent = PdfQuestionAgent(runtime)
     fast_api.state.user_spec_agent = UserSpecAgent(runtime)
     fast_api.state.execution_planning_agent = ExecutionPlanningAgent(runtime)
+    fast_api.state.math_auditor_agent = MathAuditorAgent(runtime)
     fast_api.state.form_analyser_agent = FormAnalyserAgent(runtime)
     fast_api.state.form_filler_agent = FormFillerAgent(runtime)
     fast_api.state.document_extractor_agent = DocumentExtractorAgent(runtime)
+    tracer_provider = setup_posthog_tracking(settings)
+    if tracer_provider:
+        Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider))
     yield
+    if tracer_provider:
+        tracer_provider.shutdown()
 
 
 app = FastAPI(title="Stirling AI Engine", lifespan=lifespan, version="0.1.0")
+app.add_middleware(UserIdMiddleware)
 app.include_router(orchestrator_router)
 app.include_router(pdf_edit_router)
 app.include_router(pdf_question_router)
 app.include_router(agent_draft_router)
 app.include_router(execution_router)
+app.include_router(rag_router)
+app.include_router(ledger_router)
 app.include_router(form_fill_router)
 
 

engine/src/stirling/api/dependencies.py

@@ -12,6 +12,8 @@ from stirling.agents import (
     PdfQuestionAgent,
     UserSpecAgent,
 )
+from stirling.agents.ledger import MathAuditorAgent
+from stirling.rag import RagService
 from stirling.services import AppRuntime
 
 
@@ -39,6 +41,18 @@ def get_execution_planning_agent(request: Request) -> ExecutionPlanningAgent:
     return request.app.state.execution_planning_agent
 
 
+def get_rag_service(request: Request) -> RagService:
+    return request.app.state.runtime.rag_service
+
+
+def get_rag_embedding_model(request: Request) -> str:
+    return request.app.state.runtime.settings.rag_embedding_model
+
+
+def get_math_auditor_agent(request: Request) -> MathAuditorAgent:
+    return request.app.state.math_auditor_agent
+
+
 def get_form_analyser_agent(request: Request) -> FormAnalyserAgent:
     return request.app.state.form_analyser_agent
 

engine/src/stirling/api/middleware.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+
+from stirling.services.tracking import current_user_id
+
+_USER_ID_HEADER = "X-User-Id"
+
+
+class UserIdMiddleware(BaseHTTPMiddleware):
+    """Extract X-User-Id header and set it as the current user for PostHog tracking."""
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        user_id = request.headers.get(_USER_ID_HEADER)
+        if user_id:
+            token = current_user_id.set(user_id)
+            try:
+                return await call_next(request)
+            finally:
+                current_user_id.reset(token)
+        return await call_next(request)

engine/src/stirling/api/routes/__init__.py

@@ -1,15 +1,19 @@
 from .agent_drafts import router as agent_draft_router
 from .execution import router as execution_router
 from .form_fill import router as form_fill_router
+from .ledger import router as ledger_router
 from .orchestrator import router as orchestrator_router
 from .pdf_edit import router as pdf_edit_router
 from .pdf_questions import router as pdf_question_router
+from .rag import router as rag_router
 
 __all__ = [
     "agent_draft_router",
     "execution_router",
     "form_fill_router",
+    "ledger_router",
     "orchestrator_router",
     "pdf_edit_router",
     "pdf_question_router",
+    "rag_router",
 ]

engine/src/stirling/api/routes/ledger.py

@@ -0,0 +1,60 @@
+"""
+Math Auditor Agent (mathAuditorAgent) — FastAPI routes.
+
+Two internal endpoints, called only by the Java MathAuditorOrchestrator:
+
+  POST /api/v1/ai/math-auditor-agent/examine
+      Java sends a FolioManifest (cheap page classification).
+      Python returns a Requisition (what Java must extract).
+
+  POST /api/v1/ai/math-auditor-agent/deliberate
+      Java sends Evidence (fulfilled extraction results).
+      Python returns a Verdict directly.
+"""
+
+from __future__ import annotations
+
+import logging
+from decimal import Decimal, InvalidOperation
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from stirling.agents.ledger import MathAuditorAgent
+from stirling.api.dependencies import get_math_auditor_agent
+from stirling.contracts.ledger import (
+    Evidence,
+    FolioManifest,
+    Requisition,
+    Verdict,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/ai/math-auditor-agent", tags=["math-auditor-agent"])
+
+
+@router.post("/examine", response_model=Requisition)
+async def examine_endpoint(
+    manifest: FolioManifest,
+    agent: Annotated[MathAuditorAgent, Depends(get_math_auditor_agent)],
+) -> Requisition:
+    """Round 1: Java presents a FolioManifest; Python declares its Requisition."""
+    return await agent.examine(manifest)
+
+
+@router.post("/deliberate", response_model=Verdict)
+async def deliberate_endpoint(
+    evidence: Evidence,
+    agent: Annotated[MathAuditorAgent, Depends(get_math_auditor_agent)],
+    tolerance: str = Query(default="0.01"),
+) -> Verdict:
+    """Round 2: Java presents fulfilled Evidence; Python returns a Verdict."""
+    try:
+        tol = Decimal(tolerance)
+        if tol < 0:
+            raise HTTPException(status_code=400, detail="tolerance must be non-negative")
+    except InvalidOperation:
+        raise HTTPException(status_code=400, detail=f"Invalid tolerance value: {tolerance!r}")
+
+    return await agent.audit(evidence, tol)

engine/src/stirling/api/routes/rag.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+
+from stirling.api.dependencies import get_rag_embedding_model, get_rag_service
+from stirling.contracts import (
+    RagCollectionsResponse,
+    RagDeleteCollectionResponse,
+    RagIndexRequest,
+    RagIndexResponse,
+    RagSearchRequest,
+    RagSearchResponse,
+    RagSearchResultItem,
+    RagStatusResponse,
+)
+from stirling.rag import RagService
+
+router = APIRouter(prefix="/api/v1/rag", tags=["rag"])
+
+
+@router.get("/status", response_model=RagStatusResponse)
+async def rag_status(
+    rag: Annotated[RagService, Depends(get_rag_service)],
+    embedding_model: Annotated[str, Depends(get_rag_embedding_model)],
+) -> RagStatusResponse:
+    collections = await rag.list_collections()
+    return RagStatusResponse(embedding_model=embedding_model, collections=collections)
+
+
+@router.post("/index", response_model=RagIndexResponse)
+async def rag_index(
+    request: RagIndexRequest,
+    rag: Annotated[RagService, Depends(get_rag_service)],
+) -> RagIndexResponse:
+    count = await rag.index_text(
+        collection=request.collection,
+        text=request.text,
+        source=request.source,
+        metadata=request.metadata,
+    )
+    return RagIndexResponse(collection=request.collection, chunks_indexed=count)
+
+
+@router.post("/search", response_model=RagSearchResponse)
+async def rag_search(
+    request: RagSearchRequest,
+    rag: Annotated[RagService, Depends(get_rag_service)],
+) -> RagSearchResponse:
+    results = await rag.search(query=request.query, collection=request.collection, top_k=request.top_k)
+    items = [
+        RagSearchResultItem(
+            text=r.document.text,
+            source=r.document.metadata.get("source", ""),
+            chunk_id=r.document.metadata.get("chunk_index", ""),
+            score=r.score,
+        )
+        for r in results
+    ]
+    return RagSearchResponse(query=request.query, results=items)
+
+
+@router.get("/collections", response_model=RagCollectionsResponse)
+async def rag_collections(
+    rag: Annotated[RagService, Depends(get_rag_service)],
+) -> RagCollectionsResponse:
+    collections = await rag.list_collections()
+    return RagCollectionsResponse(collections=collections)
+
+
+@router.delete("/collections/{name}", response_model=RagDeleteCollectionResponse)
+async def rag_delete_collection(
+    name: str,
+    rag: Annotated[RagService, Depends(get_rag_service)],
+) -> RagDeleteCollectionResponse:
+    await rag.delete_collection(name)
+    return RagDeleteCollectionResponse(status="deleted", collection=name)