diff --git a/engine/config/.env.example b/engine/config/.env.example deleted file mode 100644 index 64fb0e67ae..0000000000 --- a/engine/config/.env.example +++ /dev/null @@ -1,9 +0,0 @@ -# Configure the model strings passed to pydantic-ai. Provider credentials are handled by -# pydantic-ai and should be set using the provider's native environment variables, for example -# ANTHROPIC_API_KEY or OPENAI_API_KEY. -STIRLING_SMART_MODEL=anthropic:claude-haiku-4-5 -STIRLING_FAST_MODEL=anthropic:claude-haiku-4-5 - -# Default output token limits applied by the engine for each model tier. -STIRLING_SMART_MODEL_MAX_TOKENS=8192 -STIRLING_FAST_MODEL_MAX_TOKENS=2048 diff --git a/engine/src/stirling/agents/__init__.py b/engine/src/stirling/agents/__init__.py index bdeebebc15..dab176552f 100644 --- a/engine/src/stirling/agents/__init__.py +++ b/engine/src/stirling/agents/__init__.py @@ -1,6 +1,9 @@ """Agent modules for Stirling AI reasoning flows.""" from .execution import ExecutionPlanningAgent +from .form_analyser import FormAnalyserAgent +from .form_fill import FormFillAgent +from .form_filler import FormFillerAgent from .orchestrator import OrchestratorAgent from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection from .pdf_questions import PdfQuestionAgent @@ -8,6 +11,9 @@ from .user_spec import UserSpecAgent __all__ = [ "ExecutionPlanningAgent", + "FormAnalyserAgent", + "FormFillAgent", + "FormFillerAgent", "OrchestratorAgent", "PdfEditAgent", "PdfEditParameterSelector", diff --git a/engine/src/stirling/agents/form_analyser.py b/engine/src/stirling/agents/form_analyser.py new file mode 100644 index 0000000000..5ecfb83ee1 --- /dev/null +++ b/engine/src/stirling/agents/form_analyser.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from pydantic_ai import Agent +from pydantic_ai.output import NativeOutput + +from stirling.contracts.form_fill import ( + FormAnalysisRequest, + FormAnalysisResponse, + FormField, +) +from stirling.services import AppRuntime + +FORM_ANALYSER_SYSTEM_PROMPT = """\ +You are a PDF form analyser. You receive field metadata from one or more PDF forms. + +Your job (analysis ONLY — do NOT fill any values): + +1. PER-FILE ROLE DETECTION: For each file, identify distinct sections and the role each \ +represents. Use field name prefixes (e.g. "Client" vs "Beneficiary"), nearby page text \ +section headers, and form context to group fields by role. Mark which role is the \ +"primary person" (the form submitter). If a form has only one section or no section \ +distinction, create a single role for it and mark it as primary. + +2. CROSS-FILE ROLE MERGING: When the same conceptual role appears in multiple files \ +(e.g. "Client" in file A and "Applicant" in file B both mean the primary form submitter), \ +merge them into a single cross_file_role with a canonical label. Use semantic understanding \ +— "Client", "Applicant", "Account Holder" may all be the same role. Only merge when \ +confident they refer to the same type of person/entity. + +3. LABEL CLEANUP: For fields with unreadable labels (numeric codes like "0021-6009", \ +technical IDs like "fld_x7q", names with brackets or underscores), find the real label \ +from the nearby page text. Use page text verbatim. Do not clean labels that are already \ +clear readable text. + +4. INTERNAL FIELD DETECTION: Identify system fields (form IDs, submission references, \ +tracking codes, barcodes, buttons, submit buttons). Return their field names in \ +skipped_field_names per file. + +STRICT RULES: +- Do NOT return any fill values. This is analysis only. +- Every file must have at least one detected role. +- cross_file_roles must cover ALL detected roles across all files. +- If a role only appears in one file, it still appears in cross_file_roles with one file_id. +- field_names_by_file must map each file_id to the field names belonging to that role in that file. +""" + +MAX_PAGE_TEXT_CHARS = 1500 + + +class FormAnalyserAgent: + def __init__(self, runtime: AppRuntime) -> None: + self.runtime = runtime + self.agent = Agent( + model=runtime.smart_model, + output_type=NativeOutput(FormAnalysisResponse), + system_prompt=FORM_ANALYSER_SYSTEM_PROMPT, + model_settings=runtime.smart_model_settings, + ) + + async def analyse(self, request: FormAnalysisRequest) -> FormAnalysisResponse: + prompt = self._build_prompt(request) + result = await self.agent.run(prompt) + return result.output + + def _build_prompt(self, request: FormAnalysisRequest) -> str: + sections = [] + for file_set in request.files: + page_texts: dict[str, int] = {} + field_page_map: dict[str, int] = {} + for field in file_set.form_fields: + if field.nearby_page_text: + text = field.nearby_page_text + if text not in page_texts: + page_texts[text] = len(page_texts) + field_page_map[field.name] = page_texts[text] + + page_section = "\n".join( + f" [Page {pid}]: {text[:MAX_PAGE_TEXT_CHARS]}" for text, pid in page_texts.items() + ) + fields_text = "\n".join(self._format_field(f, field_page_map.get(f.name)) for f in file_set.form_fields) + sections.append( + f"=== FILE: {file_set.file_name} (id={file_set.file_id}) ===\n" + f"Page texts:\n{page_section}\n\n" + f"Fields:\n{fields_text}" + ) + + return ( + f"Analyse {len(request.files)} form(s):\n\n" + + "\n\n".join(sections) + + "\n\nDetect roles per file, merge matching roles across files, " + "clean unreadable labels, identify internal/system fields." + ) + + def _format_field(self, field: FormField, page_id: int | None) -> str: + parts = [f"- name={field.name}, type={field.type}"] + if field.label: + parts.append(f" label={field.label}") + if field.tooltip: + parts.append(f" tooltip={field.tooltip}") + if field.options: + parts.append(f" options={field.options}") + if field.required: + parts.append(" required=true") + if field.read_only: + parts.append(" readOnly=true") + if page_id is not None: + parts.append(f" page={page_id}") + return "\n".join(parts) diff --git a/engine/src/stirling/agents/form_fill.py b/engine/src/stirling/agents/form_fill.py new file mode 100644 index 0000000000..c6f28764f3 --- /dev/null +++ b/engine/src/stirling/agents/form_fill.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from pydantic_ai import Agent +from pydantic_ai.output import NativeOutput + +from stirling.contracts import ConversationMessage +from stirling.contracts.form_fill import ( + DocumentExtractionRequest, + DocumentExtractionResponse, + FormField, + FormFillClarificationResponse, + FormFillRequest, + FormFillResponse, + FormFillResultResponse, + KnowledgeUpdateResponse, + MultiProfileExtractionResponse, + RoleConfirmationResponse, + RoleDetectionResult, +) +from stirling.services import AppRuntime + +FIELD_MATCHER_SYSTEM_PROMPT = """\ +You are a PDF form field matcher. You receive a list of PDF form fields and a knowledge \ +dictionary of confirmed user information. + +Your job: +1. ROLE DETECTION: Analyze the form to identify distinct sections and the role each represents. \ +Use field name prefixes (e.g. "Client" vs "Beneficiary"), nearby page text section headers, \ +and form context to group fields by role. Return this in role_detection. + +2. CONFIDENCE SCORING for primary_confidence (0.0 to 1.0): + - 0.9-1.0: Clear primary-person keywords in field names or page text (Client, Applicant, \ +Employee, Patient, Account Holder, Policyholder, Tenant, Borrower) AND only one plausible \ +primary section. + - 0.7-0.9: Section headers suggest a primary person but terminology is ambiguous (e.g. \ +"Party A" vs "Party B"), or the form title suggests the user might be an unusual role \ +(e.g. claim form where user = claimant, not policyholder). + - 0.5-0.7: Multiple sections could plausibly be the primary person, or generic labels \ +like "Name" without section context. + - Below 0.5: No section structure detected, or all sections appear to be for third parties. + In confidence_reasoning, state which specific signals you used. + If the form has only one section or no section distinction, set primary_confidence to 1.0. + +3. FIELD MATCHING: Match form fields to knowledge entries where the field is clearly asking \ +for the same information, just worded differently. ONLY fill fields in the detected primary \ +person section. Return these as filled_fields. + +4. INTERNAL FIELDS: Identify system fields (form IDs, submission references, tracking codes, \ +barcodes, buttons, submit buttons). Return their field names in skipped_field_names. + +5. LABEL CLEANUP: For fields with unreadable labels (numeric codes, technical IDs, brackets, \ +underscores), find the real label from nearby page text and return it in cleaned_labels. \ +Use page text verbatim. Do not clean labels that are already readable. + +{role_override_instruction} + +STRICT RULES: +- ONLY fill fields belonging to the primary person section. +- ONLY fill a field if a knowledge entry is clearly the same information. +- For checkbox fields: value must be "Yes" or "Off". +- For radio/combobox/listbox fields: value MUST be one of the field's options. +- Skip read-only fields entirely. +- role_detection is REQUIRED in every response. +""" + +ROLE_OVERRIDE_INSTRUCTION = """\ +ROLE OVERRIDE: The user has confirmed they are the "{role_label}" in this form. \ +Treat all fields in that section as the primary person section unconditionally. \ +Set primary_confidence to 1.0.""" + +DOCUMENT_EXTRACTOR_SYSTEM_PROMPT = """\ +You extract structured personal information from document text. The user has uploaded a document \ +(CV, ID, utility bill, etc.) and wants to store the extracted information for future form filling. + +STRICT RULES: +- Only extract information that is explicitly stated in the document. Never infer or guess. +- Use consistent snake_case keys. Preferred keys include: first_name, last_name, full_name, \ +date_of_birth, email, phone, address_line_1, address_line_2, city, state, zip_code, country, \ +job_title, company, nationality, gender, marital_status, passport_number, driver_license_number, \ +social_security_number, tax_id. +- Use additional descriptive keys when the information doesn't fit the preferred keys. +- Set the source field to describe the document type (e.g., "extracted from CV", \ +"extracted from utility bill"). +- Provide a brief summary message listing what was extracted. +""" + +MULTI_DOC_EXTRACTOR_SYSTEM_PROMPT = """\ +You receive text from one or more documents. Your job is to extract personal information and \ +determine how many distinct people are represented. + +RULES: +- Analyze all documents to identify distinct individuals. Look for different names, different \ +contact details, different employment histories, etc. +- If all documents belong to one person, return a single profile. +- If documents belong to different people, group entries by person and return separate profiles. +- For each profile, suggest a name using the person's actual name if found (e.g. "John Smith"). \ +If no name is found, use a descriptive name like "Person from document2.pdf". +- Avoid suggesting profile names that conflict with the existing_profile_names list. +- Use consistent snake_case keys: first_name, last_name, full_name, date_of_birth, email, \ +phone, address_line_1, address_line_2, city, state, zip_code, country, job_title, company, \ +nationality, gender, etc. +- Only extract information explicitly stated in the documents. Never infer or guess. +- Set the source field to indicate which document the entry came from. +""" + +CONFIDENCE_THRESHOLD_AUTO = 0.9 +CONFIDENCE_THRESHOLD_CONFIRM = 0.5 +MAX_PAGE_TEXT_CHARS = 1500 + + +class FormFillAgent: + def __init__(self, runtime: AppRuntime) -> None: + self.runtime = runtime + self.field_matcher = Agent( + model=runtime.smart_model, + output_type=NativeOutput(FormFillResultResponse), + system_prompt=FIELD_MATCHER_SYSTEM_PROMPT.format(role_override_instruction=""), + model_settings=runtime.smart_model_settings, + ) + self.document_extractor = Agent( + model=runtime.smart_model, + output_type=NativeOutput(KnowledgeUpdateResponse), + system_prompt=DOCUMENT_EXTRACTOR_SYSTEM_PROMPT, + model_settings=runtime.smart_model_settings, + ) + self.multi_doc_extractor = Agent( + model=runtime.smart_model, + output_type=NativeOutput(MultiProfileExtractionResponse), + system_prompt=MULTI_DOC_EXTRACTOR_SYSTEM_PROMPT, + model_settings=runtime.smart_model_settings, + ) + + async def handle(self, request: FormFillRequest) -> FormFillResponse: + if request.extracted_document_text and not request.form_fields: + return await self._extract_knowledge(request) + if not request.form_fields: + return FormFillClarificationResponse( + question="Please upload a PDF form or a document to extract information from.", + reason="No form fields or document text provided.", + ) + return await self._fill_form(request) + + async def _fill_form(self, request: FormFillRequest) -> FormFillResponse: + response = await self._run_fill_agent(request) + + # If role override or preference match, return as-is + if request.role_override or self._preference_matches(request, response): + return response + + # Evaluate confidence + if response.role_detection and response.role_detection.primary_confidence < CONFIDENCE_THRESHOLD_AUTO: + return RoleConfirmationResponse( + role_detection=response.role_detection, + suggested_primary=response.role_detection.primary_role_label or "Unknown", + question=self._build_confirmation_question(response.role_detection), + provisional_fills=response.filled_fields, + cleaned_labels=response.cleaned_labels, + skipped_field_names=response.skipped_field_names, + ) + + return response + + async def _run_fill_agent(self, request: FormFillRequest) -> FormFillResultResponse: + role_instruction = "" + if request.role_override: + role_instruction = ROLE_OVERRIDE_INSTRUCTION.format(role_label=request.role_override) + + prompt = self._build_fill_prompt(request) + + if role_instruction: + agent = Agent( + model=self.runtime.smart_model, + output_type=NativeOutput(FormFillResultResponse), + system_prompt=FIELD_MATCHER_SYSTEM_PROMPT.format(role_override_instruction=role_instruction), + model_settings=self.runtime.smart_model_settings, + ) + else: + agent = self.field_matcher + + result = await agent.run(prompt) + return result.output + + async def _extract_knowledge(self, request: FormFillRequest) -> KnowledgeUpdateResponse: + prompt = self._build_extraction_prompt(request) + result = await self.document_extractor.run(prompt) + return result.output + + async def extract_documents(self, request: DocumentExtractionRequest) -> DocumentExtractionResponse: + docs_text = "\n\n".join(f"--- Document: {doc.file_name} ---\n{doc.text[:10000]}" for doc in request.documents) + existing = ", ".join(request.existing_profile_names) if request.existing_profile_names else "None" + prompt = ( + f"Documents ({len(request.documents)} total):\n{docs_text}\n\n" + f"Existing profile names (avoid conflicts): {existing}\n\n" + "Identify distinct people, extract their information, and group by person." + ) + result = await self.multi_doc_extractor.run(prompt) + return result.output + + def _preference_matches(self, request: FormFillRequest, response: FormFillResultResponse) -> bool: + pref = request.knowledge.get("_role_preference", "") + if not pref or not response.role_detection: + return False + keywords = [k.strip().lower() for k in pref.split(",") if k.strip()] + primary_label = (response.role_detection.primary_role_label or "").lower() + return any(kw in primary_label for kw in keywords) + + def _build_confirmation_question(self, detection: RoleDetectionResult) -> str: + if detection.primary_confidence >= CONFIDENCE_THRESHOLD_CONFIRM: + return f"Are you the '{detection.primary_role_label}' in this form?" + role_labels = [r.role_label for r in detection.detected_roles] + return f"Which section describes you? Detected sections: {', '.join(role_labels)}" + + def _build_fill_prompt(self, request: FormFillRequest) -> str: + # Deduplicate page texts by content, truncate each + page_texts: dict[str, int] = {} + field_page_map: dict[str, int] = {} + for field in request.form_fields: + if field.nearby_page_text: + text = field.nearby_page_text + if text not in page_texts: + page_texts[text] = len(page_texts) + field_page_map[field.name] = page_texts[text] + + page_section = "\n".join(f"[Page {pid}]: {text[:MAX_PAGE_TEXT_CHARS]}" for text, pid in page_texts.items()) + + fields_text = "\n".join( + self._format_form_field(field, field_page_map.get(field.name)) for field in request.form_fields + ) + knowledge_text = ( + "\n".join(f"- {key}: {value}" for key, value in request.knowledge.items() if not key.startswith("_")) + if request.knowledge + else "No knowledge entries available." + ) + history_text = self._format_conversation_history(request.conversation_history) + return ( + f"User message: {request.user_message}\n\n" + f"Conversation history:\n{history_text}\n\n" + f"Known user information:\n{knowledge_text}\n\n" + f"Page texts:\n{page_section}\n\n" + f"Form fields:\n{fields_text}\n\n" + "Detect roles, match fields to knowledge for the primary person only. " + "Return cleaned labels for unreadable field names. " + "Return skipped_field_names for internal/system fields." + ) + + def _build_extraction_prompt(self, request: FormFillRequest) -> str: + history_text = self._format_conversation_history(request.conversation_history) + return ( + f"User message: {request.user_message}\n\n" + f"Conversation history:\n{history_text}\n\n" + f"Document text:\n{request.extracted_document_text}\n\n" + "Extract all personal information as structured key-value pairs." + ) + + def _format_form_field(self, field: FormField, page_id: int | None) -> str: + parts = [f"- name={field.name}, type={field.type}"] + if field.label: + parts.append(f" label={field.label}") + if field.tooltip: + parts.append(f" tooltip={field.tooltip}") + if field.options: + parts.append(f" options={field.options}") + if field.required: + parts.append(" required=true") + if field.read_only: + parts.append(" readOnly=true") + if page_id is not None: + parts.append(f" page={page_id}") + return "\n".join(parts) + + def _format_conversation_history(self, conversation_history: list[ConversationMessage]) -> str: + if not conversation_history: + return "None" + return "\n".join(f"- {message.role}: {message.content}" for message in conversation_history) diff --git a/engine/src/stirling/agents/form_filler.py b/engine/src/stirling/agents/form_filler.py new file mode 100644 index 0000000000..d570e07b09 --- /dev/null +++ b/engine/src/stirling/agents/form_filler.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from pydantic_ai import Agent +from pydantic_ai.output import NativeOutput + +from stirling.contracts.form_fill import ( + FormField, + FormFillBatchRequest, + FormFillBatchResponse, +) +from stirling.services import AppRuntime + +FORM_FILLER_SYSTEM_PROMPT = """\ +You are a PDF form filler. You receive form fields and a knowledge dictionary of \ +confirmed user information. Role detection has already been done — all fields given \ +to you belong to the user's section. + +Your job: Match form fields to knowledge entries and return fill values. + +STRICT RULES: +- ONLY fill a field if a knowledge entry is clearly the same information. +- For checkbox fields: value must be "Yes" or "Off". +- For radio/combobox/listbox fields: value MUST be one of the field's options. +- Skip read-only fields. +- Do NOT perform role detection — it is already done. +- Return filled_fields per file using the file_id from the request. +- If no fields match for a file, return an empty filled_fields list for that file. +""" + + +class FormFillerAgent: + def __init__(self, runtime: AppRuntime) -> None: + self.runtime = runtime + self.agent = Agent( + model=runtime.fast_model, + output_type=NativeOutput(FormFillBatchResponse), + system_prompt=FORM_FILLER_SYSTEM_PROMPT, + model_settings=runtime.fast_model_settings, + ) + + async def fill_batch(self, request: FormFillBatchRequest) -> FormFillBatchResponse: + prompt = self._build_prompt(request) + result = await self.agent.run(prompt) + return result.output + + def _build_prompt(self, request: FormFillBatchRequest) -> str: + knowledge_text = ( + "\n".join(f"- {k}: {v}" for k, v in request.knowledge.items() if not k.startswith("_")) + or "No knowledge entries." + ) + + file_sections = [] + for file_req in request.files: + fields_text = "\n".join(self._format_field(f) for f in file_req.form_fields) + file_sections.append(f"=== FILE {file_req.file_id} (role: {file_req.role_label}) ===\n{fields_text}") + + return ( + f"Known user information:\n{knowledge_text}\n\n" + + "\n\n".join(file_sections) + + "\n\nMatch fields to knowledge entries. Return filled_fields per file." + ) + + def _format_field(self, field: FormField) -> str: + parts = [f"- name={field.name}, type={field.type}"] + if field.label: + parts.append(f" label={field.label}") + if field.tooltip: + parts.append(f" tooltip={field.tooltip}") + if field.options: + parts.append(f" options={field.options}") + if field.required: + parts.append(" required=true") + return "\n".join(parts) diff --git a/engine/src/stirling/agents/orchestrator.py b/engine/src/stirling/agents/orchestrator.py index 1ca37c23f3..a023d5f940 100644 --- a/engine/src/stirling/agents/orchestrator.py +++ b/engine/src/stirling/agents/orchestrator.py @@ -6,6 +6,7 @@ from pydantic_ai import Agent from pydantic_ai.output import ToolOutput from pydantic_ai.tools import RunContext +from stirling.agents.form_fill import FormFillAgent from stirling.agents.pdf_edit import PdfEditAgent from stirling.agents.pdf_questions import PdfQuestionAgent from stirling.agents.user_spec import UserSpecAgent @@ -20,6 +21,7 @@ from stirling.contracts import ( PdfQuestionResponse, UnsupportedCapabilityResponse, ) +from stirling.contracts.form_fill import FormFillRequest, FormFillResponse from stirling.services import AppRuntime @@ -45,6 +47,11 @@ class OrchestratorAgent: name="delegate_pdf_question", description="Delegate questions about PDF contents and return the PDF question result.", ), + ToolOutput( + self.delegate_form_fill, + name="delegate_form_fill", + description="Delegate requests to fill PDF forms or extract personal information from documents for form filling.", + ), ToolOutput( self.delegate_user_spec, name="delegate_user_spec", @@ -62,6 +69,7 @@ class OrchestratorAgent: "Choose exactly one output function that best handles the request. " "Use delegate_pdf_edit for requested PDF modifications. " "Use delegate_pdf_question for questions about the contents of a PDF. " + "Use delegate_form_fill for requests to fill PDF forms or extract personal information for form filling. " "Use delegate_user_spec for requests to create or define an agent spec. " "Use unsupported_capability only when none of the other outputs fit." ), @@ -75,6 +83,10 @@ class OrchestratorAgent: ) return result.output + async def delegate_form_fill(self, ctx: RunContext[OrchestratorDeps]) -> FormFillResponse: + request = ctx.deps.request + return await FormFillAgent(ctx.deps.runtime).handle(FormFillRequest(user_message=request.user_message)) + async def delegate_pdf_edit(self, ctx: RunContext[OrchestratorDeps]) -> PdfEditResponse: request = ctx.deps.request return await PdfEditAgent(ctx.deps.runtime).handle( diff --git a/engine/src/stirling/api/app.py b/engine/src/stirling/api/app.py index d415f13d6f..e833c31241 100644 --- a/engine/src/stirling/api/app.py +++ b/engine/src/stirling/api/app.py @@ -5,10 +5,20 @@ from typing import Annotated from fastapi import Depends, FastAPI -from stirling.agents import ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, PdfQuestionAgent, UserSpecAgent +from stirling.agents import ( + ExecutionPlanningAgent, + FormAnalyserAgent, + FormFillAgent, + FormFillerAgent, + OrchestratorAgent, + PdfEditAgent, + PdfQuestionAgent, + UserSpecAgent, +) from stirling.api.routes import ( agent_draft_router, execution_router, + form_fill_router, orchestrator_router, pdf_edit_router, pdf_question_router, @@ -37,6 +47,9 @@ async def lifespan(fast_api: FastAPI): fast_api.state.pdf_question_agent = PdfQuestionAgent(runtime) fast_api.state.user_spec_agent = UserSpecAgent(runtime) fast_api.state.execution_planning_agent = ExecutionPlanningAgent(runtime) + fast_api.state.form_fill_agent = FormFillAgent(runtime) + fast_api.state.form_analyser_agent = FormAnalyserAgent(runtime) + fast_api.state.form_filler_agent = FormFillerAgent(runtime) yield @@ -46,6 +59,7 @@ app.include_router(pdf_edit_router) app.include_router(pdf_question_router) app.include_router(agent_draft_router) app.include_router(execution_router) +app.include_router(form_fill_router) @app.get("/health", response_model=HealthResponse) diff --git a/engine/src/stirling/api/dependencies.py b/engine/src/stirling/api/dependencies.py index c25e488d19..cf15312830 100644 --- a/engine/src/stirling/api/dependencies.py +++ b/engine/src/stirling/api/dependencies.py @@ -2,7 +2,16 @@ from __future__ import annotations from fastapi import Request -from stirling.agents import ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, PdfQuestionAgent, UserSpecAgent +from stirling.agents import ( + ExecutionPlanningAgent, + FormAnalyserAgent, + FormFillAgent, + FormFillerAgent, + OrchestratorAgent, + PdfEditAgent, + PdfQuestionAgent, + UserSpecAgent, +) from stirling.services import AppRuntime @@ -28,3 +37,15 @@ def get_user_spec_agent(request: Request) -> UserSpecAgent: def get_execution_planning_agent(request: Request) -> ExecutionPlanningAgent: return request.app.state.execution_planning_agent + + +def get_form_fill_agent(request: Request) -> FormFillAgent: + return request.app.state.form_fill_agent + + +def get_form_analyser_agent(request: Request) -> FormAnalyserAgent: + return request.app.state.form_analyser_agent + + +def get_form_filler_agent(request: Request) -> FormFillerAgent: + return request.app.state.form_filler_agent diff --git a/engine/src/stirling/api/routes/__init__.py b/engine/src/stirling/api/routes/__init__.py index 3a920ab5ea..bdc50e7128 100644 --- a/engine/src/stirling/api/routes/__init__.py +++ b/engine/src/stirling/api/routes/__init__.py @@ -1,5 +1,6 @@ from .agent_drafts import router as agent_draft_router from .execution import router as execution_router +from .form_fill import router as form_fill_router from .orchestrator import router as orchestrator_router from .pdf_edit import router as pdf_edit_router from .pdf_questions import router as pdf_question_router @@ -7,6 +8,7 @@ from .pdf_questions import router as pdf_question_router __all__ = [ "agent_draft_router", "execution_router", + "form_fill_router", "orchestrator_router", "pdf_edit_router", "pdf_question_router", diff --git a/engine/src/stirling/api/routes/form_fill.py b/engine/src/stirling/api/routes/form_fill.py new file mode 100644 index 0000000000..cbf5c9a9c0 --- /dev/null +++ b/engine/src/stirling/api/routes/form_fill.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import Annotated + +from fastapi import APIRouter, Depends + +from stirling.agents import FormAnalyserAgent, FormFillAgent, FormFillerAgent +from stirling.api.dependencies import get_form_analyser_agent, get_form_fill_agent, get_form_filler_agent +from stirling.contracts.form_fill import ( + DocumentExtractionRequest, + DocumentExtractionResponse, + FormAnalysisRequest, + FormAnalysisResponse, + FormFillBatchRequest, + FormFillBatchResponse, + FormFillRequest, + FormFillResponse, +) + +router = APIRouter(prefix="/api/v1/form/ai", tags=["form-fill"]) + + +@router.post("", response_model=FormFillResponse) +async def form_fill( + request: FormFillRequest, + agent: Annotated[FormFillAgent, Depends(get_form_fill_agent)], +) -> FormFillResponse: + return await agent.handle(request) + + +@router.post("/extract", response_model=DocumentExtractionResponse) +async def extract_from_documents( + request: DocumentExtractionRequest, + agent: Annotated[FormFillAgent, Depends(get_form_fill_agent)], +) -> DocumentExtractionResponse: + return await agent.extract_documents(request) + + +@router.post("/analyse", response_model=FormAnalysisResponse) +async def analyse_forms( + request: FormAnalysisRequest, + agent: Annotated[FormAnalyserAgent, Depends(get_form_analyser_agent)], +) -> FormAnalysisResponse: + return await agent.analyse(request) + + +@router.post("/fill-batch", response_model=FormFillBatchResponse) +async def fill_forms_batch( + request: FormFillBatchRequest, + agent: Annotated[FormFillerAgent, Depends(get_form_filler_agent)], +) -> FormFillBatchResponse: + return await agent.fill_batch(request) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 6988233c5d..b47b23e688 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -18,6 +18,34 @@ from .execution import ( NextExecutionAction, ToolCallExecutionAction, ) +from .form_fill import ( + AnalysedFileResult, + CleanedLabel, + CrossFileRole, + DetectedRole, + DocumentExtractionRequest, + DocumentExtractionResponse, + DocumentText, + FieldMapping, + FileFieldSet, + FileFillRequest, + FileFillResult, + FormAnalysisRequest, + FormAnalysisResponse, + FormField, + FormFillBatchRequest, + FormFillBatchResponse, + FormFillClarificationResponse, + FormFillRequest, + FormFillResponse, + FormFillResultResponse, + KnowledgeEntry, + KnowledgeUpdateResponse, + MultiProfileExtractionResponse, + ProposedProfile, + RoleConfirmationResponse, + RoleDetectionResult, +) from .health import HealthResponse from .orchestrator import OrchestratorRequest, OrchestratorResponse, SupportedCapability, UnsupportedCapabilityResponse from .pdf_edit import ( @@ -48,17 +76,43 @@ __all__ = [ "AgentSpecStep", "AiToolAgentStep", "CannotContinueExecutionAction", + "CleanedLabel", "ConversationMessage", + "DetectedRole", + "DocumentExtractionRequest", + "DocumentExtractionResponse", + "DocumentText", + "FileFieldSet", + "FileFillRequest", + "FileFillResult", + "FormAnalysisRequest", + "FormAnalysisResponse", + "AnalysedFileResult", + "CrossFileRole", + "FormFillBatchRequest", + "FormFillBatchResponse", "CompletedExecutionAction", + "FieldMapping", + "FormField", + "FormFillClarificationResponse", + "FormFillRequest", + "FormFillResponse", + "FormFillResultResponse", "EditCannotDoResponse", "EditClarificationRequest", "EditPlanResponse", "ExecutionContext", "ExecutionStepResult", "HealthResponse", + "KnowledgeEntry", + "KnowledgeUpdateResponse", + "MultiProfileExtractionResponse", "NextExecutionAction", "OrchestratorRequest", "OrchestratorResponse", + "RoleConfirmationResponse", + "RoleDetectionResult", + "ProposedProfile", "PdfEditRequest", "PdfEditResponse", "PdfQuestionAnswerResponse", diff --git a/engine/src/stirling/contracts/form_fill.py b/engine/src/stirling/contracts/form_fill.py new file mode 100644 index 0000000000..0fbf66c281 --- /dev/null +++ b/engine/src/stirling/contracts/form_fill.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field + +from stirling.models import ApiModel + +from .common import ConversationMessage + + +class FormField(ApiModel): + """PDF form field metadata for AI reasoning. Mirrors Java FormFieldWithCoordinates without widget coordinates.""" + + name: str + label: str | None = None + type: str + value: str | None = None + options: list[str] | None = None + display_options: list[str] | None = None + required: bool = False + read_only: bool = False + multi_select: bool = False + multiline: bool = False + tooltip: str | None = None + nearby_page_text: str | None = None + + +class FormFillRequest(ApiModel): + user_message: str + conversation_history: list[ConversationMessage] = Field(default_factory=list) + form_fields: list[FormField] = Field(default_factory=list) + knowledge: dict[str, str] = Field(default_factory=dict) + extracted_document_text: str | None = None + role_override: str | None = None + + +class FieldMapping(ApiModel): + field_name: str + knowledge_key: str + value: str + + +class CleanedLabel(ApiModel): + field_name: str + label: str + + +class KnowledgeEntry(ApiModel): + key: str + value: str + source: str + + +class DocumentText(ApiModel): + file_name: str + text: str + + +class DocumentExtractionRequest(ApiModel): + documents: list[DocumentText] + existing_profile_names: list[str] = Field(default_factory=list) + + +class ProposedProfile(ApiModel): + suggested_name: str + entries: list[KnowledgeEntry] + source_documents: list[str] + + +class MultiProfileExtractionResponse(ApiModel): + outcome: Literal["multi_profile_extraction"] = "multi_profile_extraction" + proposed_profiles: list[ProposedProfile] + message: str + + +class DetectedRole(ApiModel): + role_label: str + field_names: list[str] + is_primary_person: bool + + +class RoleDetectionResult(ApiModel): + detected_roles: list[DetectedRole] + primary_role_label: str | None = None + primary_confidence: float = Field(ge=0.0, le=1.0) + confidence_reasoning: str + + +class FormFillResultResponse(ApiModel): + outcome: Literal["fill_result"] = "fill_result" + filled_fields: list[FieldMapping] + cleaned_labels: list[CleanedLabel] = Field(default_factory=list) + skipped_field_names: list[str] = Field(default_factory=list) + role_detection: RoleDetectionResult | None = None + message: str + + +class RoleConfirmationResponse(ApiModel): + outcome: Literal["role_confirmation_needed"] = "role_confirmation_needed" + role_detection: RoleDetectionResult + suggested_primary: str + question: str + provisional_fills: list[FieldMapping] + cleaned_labels: list[CleanedLabel] = Field(default_factory=list) + skipped_field_names: list[str] = Field(default_factory=list) + + +class KnowledgeUpdateResponse(ApiModel): + outcome: Literal["knowledge_update"] = "knowledge_update" + proposed_entries: list[KnowledgeEntry] + message: str + + +class FormFillClarificationResponse(ApiModel): + outcome: Literal["form_fill_clarification"] = "form_fill_clarification" + question: str + reason: str + + +# --- Form Analysis (multi-file) --- + + +class FileFieldSet(ApiModel): + file_id: str + file_name: str + form_fields: list[FormField] = Field(default_factory=list) + + +class FormAnalysisRequest(ApiModel): + files: list[FileFieldSet] + + +class AnalysedFileResult(ApiModel): + file_id: str + file_name: str + detected_roles: list[DetectedRole] + cleaned_labels: list[CleanedLabel] = Field(default_factory=list) + skipped_field_names: list[str] = Field(default_factory=list) + + +class CrossFileRole(ApiModel): + role_label: str + file_ids: list[str] + field_names_by_file: dict[str, list[str]] + is_primary_person: bool + + +class FormAnalysisResponse(ApiModel): + per_file: list[AnalysedFileResult] + cross_file_roles: list[CrossFileRole] + message: str + + +# --- Batch Fill --- + + +class FileFillRequest(ApiModel): + file_id: str + form_fields: list[FormField] = Field(default_factory=list) + role_label: str + + +class FormFillBatchRequest(ApiModel): + files: list[FileFillRequest] + knowledge: dict[str, str] = Field(default_factory=dict) + + +class FileFillResult(ApiModel): + file_id: str + filled_fields: list[FieldMapping] + + +class FormFillBatchResponse(ApiModel): + outcome: Literal["batch_fill_result"] = "batch_fill_result" + per_file: list[FileFillResult] + message: str + + +DocumentExtractionResponse = Annotated[ + KnowledgeUpdateResponse | MultiProfileExtractionResponse, + Field(discriminator="outcome"), +] + + +FormFillResponse = Annotated[ + FormFillResultResponse | RoleConfirmationResponse | KnowledgeUpdateResponse | FormFillClarificationResponse, + Field(discriminator="outcome"), +] diff --git a/engine/src/stirling/contracts/orchestrator.py b/engine/src/stirling/contracts/orchestrator.py index 563341880c..d3e819e19e 100644 --- a/engine/src/stirling/contracts/orchestrator.py +++ b/engine/src/stirling/contracts/orchestrator.py @@ -9,6 +9,7 @@ from stirling.models import ApiModel from .agent_drafts import AgentDraftResponse from .execution import NextExecutionAction +from .form_fill import FormFillResponse from .pdf_edit import PdfEditResponse from .pdf_questions import PdfQuestionResponse @@ -20,6 +21,7 @@ class SupportedCapability(StrEnum): AGENT_DRAFT = "agent_draft" AGENT_REVISE = "agent_revise" AGENT_NEXT_ACTION = "agent_next_action" + FORM_FILL = "form_fill" class OrchestratorRequest(ApiModel): @@ -34,6 +36,11 @@ class UnsupportedCapabilityResponse(ApiModel): OrchestratorResponse = Annotated[ - PdfEditResponse | PdfQuestionResponse | AgentDraftResponse | NextExecutionAction | UnsupportedCapabilityResponse, + PdfEditResponse + | PdfQuestionResponse + | FormFillResponse + | AgentDraftResponse + | NextExecutionAction + | UnsupportedCapabilityResponse, Field(discriminator="outcome"), ] diff --git a/engine/tests/test_form_fill_agent.py b/engine/tests/test_form_fill_agent.py new file mode 100644 index 0000000000..edc86ad059 --- /dev/null +++ b/engine/tests/test_form_fill_agent.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import pytest + +from stirling.agents import FormFillAgent +from stirling.config import AppSettings +from stirling.contracts.form_fill import ( + DetectedRole, + FieldMapping, + FormField, + FormFillClarificationResponse, + FormFillRequest, + FormFillResultResponse, + KnowledgeEntry, + KnowledgeUpdateResponse, + RoleConfirmationResponse, + RoleDetectionResult, +) +from stirling.services import build_runtime + + +def build_test_settings() -> AppSettings: + return AppSettings( + smart_model_name="test", + fast_model_name="test", + smart_model_max_tokens=8192, + fast_model_max_tokens=2048, + ) + + +class StubFormFillAgent(FormFillAgent): + def __init__( + self, + fill_response: FormFillResultResponse | None = None, + extraction_response: KnowledgeUpdateResponse | None = None, + ) -> None: + super().__init__(build_runtime(build_test_settings())) + self._fill_response = fill_response + self._extraction_response = extraction_response + + async def _run_fill_agent(self, request: FormFillRequest) -> FormFillResultResponse: + assert self._fill_response is not None + return self._fill_response + + async def _extract_knowledge(self, request: FormFillRequest) -> KnowledgeUpdateResponse: + assert self._extraction_response is not None + return self._extraction_response + + +@pytest.mark.anyio +async def test_form_fill_returns_clarification_when_no_fields_or_document() -> None: + agent = FormFillAgent(build_runtime(build_test_settings())) + + response = await agent.handle(FormFillRequest(user_message="Fill this form")) + + assert isinstance(response, FormFillClarificationResponse) + + +@pytest.mark.anyio +async def test_form_fill_routes_to_fill_when_fields_provided() -> None: + expected = FormFillResultResponse( + filled_fields=[FieldMapping(field_name="name", knowledge_key="full_name", value="John Doe")], + role_detection=RoleDetectionResult( + detected_roles=[DetectedRole(role_label="Applicant", field_names=["name"], is_primary_person=True)], + primary_role_label="Applicant", + primary_confidence=1.0, + confidence_reasoning="Single section form.", + ), + message="Filled 1 field.", + ) + agent = StubFormFillAgent(fill_response=expected) + + response = await agent.handle( + FormFillRequest( + user_message="Fill this form", + form_fields=[FormField(name="name", type="text", label="Full Name")], + knowledge={"full_name": "John Doe"}, + ) + ) + + assert isinstance(response, FormFillResultResponse) + assert len(response.filled_fields) == 1 + assert response.filled_fields[0].value == "John Doe" + + +@pytest.mark.anyio +async def test_form_fill_routes_to_extraction_when_document_text_provided() -> None: + expected = KnowledgeUpdateResponse( + proposed_entries=[KnowledgeEntry(key="full_name", value="John Doe", source="extracted from CV")], + message="Extracted 1 entry.", + ) + agent = StubFormFillAgent(extraction_response=expected) + + response = await agent.handle( + FormFillRequest( + user_message="Extract my info", + extracted_document_text="John Doe, Software Engineer, john@example.com", + ) + ) + + assert isinstance(response, KnowledgeUpdateResponse) + assert len(response.proposed_entries) == 1 + assert response.proposed_entries[0].key == "full_name" + + +@pytest.mark.anyio +async def test_form_fill_returns_empty_filled_for_unknown_info() -> None: + expected = FormFillResultResponse( + filled_fields=[], + role_detection=RoleDetectionResult( + detected_roles=[DetectedRole(role_label="Applicant", field_names=["ssn"], is_primary_person=True)], + primary_role_label="Applicant", + primary_confidence=1.0, + confidence_reasoning="Single section form.", + ), + message="Could not fill any fields.", + ) + agent = StubFormFillAgent(fill_response=expected) + + response = await agent.handle( + FormFillRequest( + user_message="Fill this form", + form_fields=[FormField(name="ssn", type="text", label="Social Security Number")], + knowledge={}, + ) + ) + + assert isinstance(response, FormFillResultResponse) + assert len(response.filled_fields) == 0 + + +@pytest.mark.anyio +async def test_form_fill_prefers_fill_when_both_fields_and_document_provided() -> None: + expected = FormFillResultResponse( + filled_fields=[FieldMapping(field_name="email", knowledge_key="email", value="john@example.com")], + role_detection=RoleDetectionResult( + detected_roles=[DetectedRole(role_label="Applicant", field_names=["email"], is_primary_person=True)], + primary_role_label="Applicant", + primary_confidence=1.0, + confidence_reasoning="Single section form.", + ), + message="Filled 1 field.", + ) + agent = StubFormFillAgent(fill_response=expected) + + response = await agent.handle( + FormFillRequest( + user_message="Fill this form", + form_fields=[FormField(name="email", type="text", label="Email")], + knowledge={"email": "john@example.com"}, + extracted_document_text="Some document text", + ) + ) + + assert isinstance(response, FormFillResultResponse) + + +@pytest.mark.anyio +async def test_low_confidence_returns_role_confirmation() -> None: + expected = FormFillResultResponse( + filled_fields=[FieldMapping(field_name="client_name", knowledge_key="full_name", value="John Doe")], + role_detection=RoleDetectionResult( + detected_roles=[ + DetectedRole(role_label="Client", field_names=["client_name"], is_primary_person=True), + DetectedRole(role_label="Beneficiary", field_names=["beneficiary_name"], is_primary_person=False), + ], + primary_role_label="Client", + primary_confidence=0.7, + confidence_reasoning="Field prefixes suggest Client is primary.", + ), + message="Filled 1 field.", + ) + agent = StubFormFillAgent(fill_response=expected) + + response = await agent.handle( + FormFillRequest( + user_message="Fill this form", + form_fields=[ + FormField(name="client_name", type="text", label="Client Name"), + FormField(name="beneficiary_name", type="text", label="Beneficiary Name"), + ], + knowledge={"full_name": "John Doe"}, + ) + ) + + assert isinstance(response, RoleConfirmationResponse) + assert response.suggested_primary == "Client" + assert len(response.provisional_fills) == 1 + + +@pytest.mark.anyio +async def test_preference_match_skips_confirmation() -> None: + expected = FormFillResultResponse( + filled_fields=[FieldMapping(field_name="client_name", knowledge_key="full_name", value="John Doe")], + role_detection=RoleDetectionResult( + detected_roles=[ + DetectedRole(role_label="Client", field_names=["client_name"], is_primary_person=True), + DetectedRole(role_label="Beneficiary", field_names=["beneficiary_name"], is_primary_person=False), + ], + primary_role_label="Client", + primary_confidence=0.7, + confidence_reasoning="Field prefixes suggest Client is primary.", + ), + message="Filled 1 field.", + ) + agent = StubFormFillAgent(fill_response=expected) + + response = await agent.handle( + FormFillRequest( + user_message="Fill this form", + form_fields=[ + FormField(name="client_name", type="text", label="Client Name"), + FormField(name="beneficiary_name", type="text", label="Beneficiary Name"), + ], + knowledge={"full_name": "John Doe", "_role_preference": "client"}, + ) + ) + + # Preference matches "Client", so no confirmation needed + assert isinstance(response, FormFillResultResponse) diff --git a/engine/tests/test_stirling_api.py b/engine/tests/test_stirling_api.py index 9191eeb441..0f2aa28f90 100644 --- a/engine/tests/test_stirling_api.py +++ b/engine/tests/test_stirling_api.py @@ -3,6 +3,7 @@ from fastapi.testclient import TestClient from stirling.api import app from stirling.api.dependencies import ( get_execution_planning_agent, + get_form_fill_agent, get_orchestrator_agent, get_pdf_edit_agent, get_pdf_question_agent, @@ -24,6 +25,10 @@ from stirling.contracts import ( PdfQuestionRequest, UnsupportedCapabilityResponse, ) +from stirling.contracts.form_fill import ( + FormFillClarificationResponse, + FormFillRequest, +) from stirling.models.tool_models import RotateParams @@ -67,6 +72,14 @@ class StubUserSpecAgent: return AgentRevisionResponse(draft=request.current_draft) +class StubFormFillAgent: + async def handle(self, request: FormFillRequest) -> FormFillClarificationResponse: + return FormFillClarificationResponse( + question="Please upload a PDF form.", + reason=request.user_message, + ) + + class StubExecutionPlanningAgent: async def next_action(self, request: AgentExecutionRequest) -> CannotContinueExecutionAction: return CannotContinueExecutionAction(reason=str(request.current_step_index)) @@ -95,6 +108,10 @@ def override_user_spec_agent() -> StubUserSpecAgent: return StubUserSpecAgent() +def override_form_fill_agent() -> StubFormFillAgent: + return StubFormFillAgent() + + def override_execution_agent() -> StubExecutionPlanningAgent: return StubExecutionPlanningAgent() @@ -105,6 +122,7 @@ app.dependency_overrides[get_pdf_edit_agent] = override_pdf_edit_agent app.dependency_overrides[get_pdf_question_agent] = override_pdf_question_agent app.dependency_overrides[get_user_spec_agent] = override_user_spec_agent app.dependency_overrides[get_execution_planning_agent] = override_execution_agent +app.dependency_overrides[get_form_fill_agent] = override_form_fill_agent def test_health_route() -> None: @@ -166,6 +184,13 @@ def test_agent_revise_route() -> None: assert response.json()["outcome"] == "draft" +def test_form_fill_route() -> None: + response = client.post("/api/v1/form/ai", json={"userMessage": "fill my form"}) + + assert response.status_code == 200 + assert response.json()["outcome"] == "form_fill_clarification" + + def test_next_action_route() -> None: response = client.post( "/api/v1/agents/next-action", diff --git a/engine/tests/test_stirling_contracts.py b/engine/tests/test_stirling_contracts.py index 7cf7df974c..c3fabee9ce 100644 --- a/engine/tests/test_stirling_contracts.py +++ b/engine/tests/test_stirling_contracts.py @@ -13,6 +13,17 @@ from stirling.contracts import ( PdfQuestionAnswerResponse, ToolOperationStep, ) +from stirling.contracts.form_fill import ( + DetectedRole, + FieldMapping, + FormFillClarificationResponse, + FormFillRequest, + FormFillResultResponse, + KnowledgeEntry, + KnowledgeUpdateResponse, + RoleConfirmationResponse, + RoleDetectionResult, +) from stirling.models.tool_models import OperationId, RotateParams @@ -66,6 +77,52 @@ def clear_settings_cache() -> Iterator[None]: load_settings.cache_clear() +def test_form_fill_request_serializes_to_camel_case() -> None: + request = FormFillRequest(user_message="Fill this", knowledge={"name": "John"}) + data = request.model_dump(by_alias=True) + + assert "userMessage" in data + assert "formFields" in data + assert "extractedDocumentText" in data + + +def test_form_fill_result_response_discriminator() -> None: + fill = FormFillResultResponse( + filled_fields=[FieldMapping(field_name="name", knowledge_key="full_name", value="John")], + message="Done.", + ) + assert fill.outcome == "fill_result" + + update = KnowledgeUpdateResponse( + proposed_entries=[KnowledgeEntry(key="name", value="John", source="CV")], + message="Extracted.", + ) + assert update.outcome == "knowledge_update" + + clarification = FormFillClarificationResponse(question="What?", reason="Missing info.") + assert clarification.outcome == "form_fill_clarification" + + +def test_role_confirmation_response() -> None: + confirmation = RoleConfirmationResponse( + role_detection=RoleDetectionResult( + detected_roles=[ + DetectedRole(role_label="Client", field_names=["name"], is_primary_person=True), + DetectedRole(role_label="Beneficiary", field_names=["ben_name"], is_primary_person=False), + ], + primary_role_label="Client", + primary_confidence=0.75, + confidence_reasoning="Field prefixes.", + ), + suggested_primary="Client", + question="Are you the Client?", + provisional_fills=[FieldMapping(field_name="name", knowledge_key="full_name", value="John")], + ) + assert confirmation.outcome == "role_confirmation_needed" + assert confirmation.suggested_primary == "Client" + assert len(confirmation.provisional_fills) == 1 + + def test_app_settings_accepts_model_configuration() -> None: settings = AppSettings( smart_model_name="claude-sonnet-4-5-20250929", diff --git a/frontend/public/locales/en-GB/translation.toml b/frontend/public/locales/en-GB/translation.toml index c35791ca69..dac1133ba0 100644 --- a/frontend/public/locales/en-GB/translation.toml +++ b/frontend/public/locales/en-GB/translation.toml @@ -3929,6 +3929,10 @@ title = "Automate" desc = "Fill PDF form fields interactively with a visual editor" title = "Fill Form" +[home.aiFormFill] +desc = "Use AI to automatically fill PDF form fields from your knowledge base" +title = "AI Form Fill" + [home.autoRename] desc = "Auto renames a PDF file based on its detected header" tags = "auto-detect,header-based,organize,relabel,auto rename,automatic rename,smart rename,rename by content,filename,file naming,detect title" @@ -8527,3 +8531,27 @@ description = "Google API Key for Google Picker API from Google Cloud Console" [provider.googledrive.appId] label = "App ID" description = "Google Drive App ID from Google Cloud Console" + +[aiFormFill] +title = "AI Form Fill" +setupDesc = "Upload a PDF form and let AI fill it using your stored knowledge." +noFile = "Open a PDF form to get started." +autoFillButton = "Auto-Fill with AI" +filling = "AI is filling your form..." +applying = "Generating filled PDF..." +filledCount = "Filled {{filled}} of {{total}} fields" +filledSection = "Filled Fields" +unfilledSection = "Needs Your Input" +saveApply = "Save & Apply" +reset = "Start Over" +knowledgeSummary = "{{count}} entries stored" +knowledgeEmpty = "No knowledge stored yet" +knowledgeClear = "Clear All" +editHint = "Edit remaining fields directly on the PDF or in the form sidebar." +roleConfirmQuestion = "Are you the \"{{role}}\" in this form?" +roleSelectQuestion = "Which section is yours?" +rememberRole = "Remember this for similar forms" +notMe = "No, let me choose" +yesConfirm = "Yes, fill as {{role}}" +fillAsRole = "Fill as {{role}}" +rerun = "Re-run AI Fill" diff --git a/frontend/src/core/data/useTranslatedToolRegistry.tsx b/frontend/src/core/data/useTranslatedToolRegistry.tsx index 3c021d0306..6469ec471f 100644 --- a/frontend/src/core/data/useTranslatedToolRegistry.tsx +++ b/frontend/src/core/data/useTranslatedToolRegistry.tsx @@ -40,6 +40,7 @@ import SingleLargePage from "@app/tools/SingleLargePage"; import PageLayout from "@app/tools/PageLayout"; import UnlockPdfForms from "@app/tools/UnlockPdfForms"; import FormFill from "@app/tools/formFill/FormFill"; +import AiFormFill from "@app/tools/aiFormFill/AiFormFill"; import RemoveCertificateSign from "@app/tools/RemoveCertificateSign"; import RemoveImage from "@app/tools/RemoveImage"; import CertSign from "@app/tools/CertSign"; @@ -375,6 +376,19 @@ export function useTranslatedToolCatalog(): TranslatedToolCatalog { supportsAutomate: false, synonyms: ['form', 'fill', 'fillable', 'input', 'field', 'acroform'], }, + aiFormFill: { + icon: , + name: t('home.aiFormFill.title', 'AI Form Fill'), + component: AiFormFill, + description: t('home.aiFormFill.desc', 'Use AI to automatically fill PDF form fields from your knowledge base'), + categoryId: ToolCategoryId.STANDARD_TOOLS, + subcategoryId: SubcategoryId.GENERAL, + workbench: 'viewer' as const, + endpoints: [], + automationSettings: null, + supportsAutomate: false, + synonyms: ['ai', 'auto', 'form', 'fill', 'smart', 'intelligent', 'knowledge'], + }, changePermissions: { icon: , name: t("home.changePermissions.title", "Change Permissions"), diff --git a/frontend/src/core/tools/aiFormFill/AiFormFill.module.css b/frontend/src/core/tools/aiFormFill/AiFormFill.module.css new file mode 100644 index 0000000000..1185c67f65 --- /dev/null +++ b/frontend/src/core/tools/aiFormFill/AiFormFill.module.css @@ -0,0 +1,39 @@ +.root { + display: flex; + flex-direction: column; + height: 100%; + overflow: hidden; +} + +.header { + padding: 0.75rem; + border-bottom: 1px solid var(--mantine-color-default-border); + flex-shrink: 0; +} + +.content { + flex: 1; + overflow-y: auto; + padding: 0.75rem; +} + +.footer { + padding: 0.75rem; + border-top: 1px solid var(--mantine-color-default-border); + flex-shrink: 0; +} + +.fieldCard { + padding: 0.5rem; + border-radius: var(--mantine-radius-sm); + border: 1px solid var(--mantine-color-default-border); +} + +.filledField { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.375rem 0.5rem; + border-radius: var(--mantine-radius-sm); + background: var(--mantine-color-green-light); +} diff --git a/frontend/src/core/tools/aiFormFill/AiFormFill.tsx b/frontend/src/core/tools/aiFormFill/AiFormFill.tsx new file mode 100644 index 0000000000..f2212b0428 --- /dev/null +++ b/frontend/src/core/tools/aiFormFill/AiFormFill.tsx @@ -0,0 +1,472 @@ +/** + * AI Form Fill tool — analyses forms (role detection, label cleanup), + * then fills using profiles. Supports multi-file bulk operations. + */ +import { useMemo, useCallback, useState } from 'react'; +import { + Stack, + Text, + Button, + Progress, + Loader, + Alert, + Badge, + Divider, + Group, + Select, + SegmentedControl, + Checkbox, +} from '@mantine/core'; +import AutoAwesomeIcon from '@mui/icons-material/AutoAwesome'; +import AnalyticsIcon from '@mui/icons-material/Analytics'; +import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import WarningAmberIcon from '@mui/icons-material/WarningAmber'; +import { useTranslation } from 'react-i18next'; +import { useFileState } from '@app/contexts/FileContext'; +import { isStirlingFile } from '@app/types/fileContext'; +import type { BaseToolProps, ToolComponent } from '@app/types/tool'; +import { useKnowledgeStore } from './useKnowledgeStore'; +import { useFormAnalysis } from './useFormAnalysis'; +import { useBatchFormFillFlow } from './useBatchFormFillFlow'; +import { ProfileManagementModal } from './ProfileManagementModal'; +import { RoleSection } from './RoleSection'; +import { FillPreview } from './FillPreview'; +import { usePassiveLearning } from './usePassiveLearning'; +import { checkExpiryDates, checkCrossFormConsistency } from './workflowTemplates'; +import type { CleanedLabel } from './types'; +import styles from './AiFormFill.module.css'; + +const AiFormFill = (_props: BaseToolProps) => { + const { t } = useTranslation(); + const { selectors, state: fileState } = useFileState(); + const knowledge = useKnowledgeStore(); + const analysis = useFormAnalysis(knowledge); + const batchFill = useBatchFormFillFlow(analysis, knowledge); + const passiveLearning = usePassiveLearning(); + const [modalOpened, setModalOpened] = useState(false); + const [viewMode, setViewMode] = useState('roles'); + + // Expiry warnings + const expiryWarnings = useMemo( + () => checkExpiryDates(knowledge.entityStore.entities), + [knowledge.entityStore.entities] + ); + + const allFiles = useMemo(() => { + const files = selectors.getFiles(); + return files.filter((f) => isStirlingFile(f)); + }, [selectors, fileState.files.ids]); + + const fileNames = useMemo(() => { + const map: Record = {}; + for (const f of allFiles) { + if (isStirlingFile(f)) map[f.fileId] = f.name; + } + return map; + }, [allFiles]); + + const handleAnalyse = useCallback(() => { + if (allFiles.length > 0) { + analysis.analyseAllFiles(allFiles as any); + } + }, [allFiles, analysis.analyseAllFiles]); + + const handleFill = useCallback(() => { + if (allFiles.length > 0) { + batchFill.fillAllFiles(allFiles as any); + } + }, [allFiles, batchFill.fillAllFiles]); + + const handleReset = useCallback(() => { + analysis.reset(); + batchFill.reset(); + }, [analysis.reset, batchFill.reset]); + + const modal = setModalOpened(false)} knowledge={knowledge} />; + + // Build lookup maps from analysis + const cleanedLabelsByFile = useMemo(() => { + const map: Record = {}; + for (const pf of analysis.analysis?.perFile || []) { + map[pf.fileId] = pf.cleanedLabels; + } + return map; + }, [analysis.analysis]); + + const skippedFieldsByFile = useMemo(() => { + const map: Record> = {}; + for (const pf of analysis.analysis?.perFile || []) { + map[pf.fileId] = new Set(pf.skippedFieldNames); + } + return map; + }, [analysis.analysis]); + + // ── Idle / Setup ── + if (analysis.phase === 'idle' || analysis.phase === 'error') { + return ( + <>{modal} +
+
+ + + {t('aiFormFill.title', 'AI Form Fill')} + + + v && knowledge.entityStore.setDefaultEntity(v)} + searchable + styles={{ input: { fontSize: '0.8125rem' } }} + style={{ flex: 1 }} + /> + + + + {roles.length} role{roles.length !== 1 ? 's' : ''} detected across {allFiles.length} file{allFiles.length !== 1 ? 's' : ''} + + {analysis.analysis?.message && ( + {analysis.analysis.message} + )} + +
+ +
+ + {viewMode === 'roles' ? ( + // Role-grouped view + roles.map((role) => ( + analysis.setRoleProfile(role.roleLabel, id)} + fieldsByFile={analysis.fieldsByFile} + cleanedLabelsByFile={cleanedLabelsByFile} + skippedFieldsByFile={skippedFieldsByFile} + fileNames={fileNames} + fileRoleOverrides={analysis.fileRoleOverrides} + onFileOverride={(fileId, entityId) => + entityId + ? analysis.setFileRoleOverride(fileId, role.roleLabel, entityId) + : analysis.clearFileRoleOverride(fileId, role.roleLabel) + } + /> + )) + ) : ( + // Original order — flat field list per file + allFiles.map((file) => { + if (!isStirlingFile(file)) return null; + const fields = analysis.fieldsByFile[file.fileId] || []; + const skipped = skippedFieldsByFile[file.fileId] || new Set(); + const labels = cleanedLabelsByFile[file.fileId] || []; + const labelMap: Record = {}; + for (const cl of labels) labelMap[cl.fieldName] = cl.label; + + return ( + + {file.name} + {fields + .filter((f) => !skipped.has(f.name) && !f.readOnly) + .map((f) => ( + + {labelMap[f.name] || f.label || f.name} + + ))} + + + ); + }) + )} + +
+ +
+ + + + +
+
+ + ); +}; + +export default AiFormFill as ToolComponent; diff --git a/frontend/src/core/tools/aiFormFill/EntitiesTab.tsx b/frontend/src/core/tools/aiFormFill/EntitiesTab.tsx new file mode 100644 index 0000000000..4e3fe8837f --- /dev/null +++ b/frontend/src/core/tools/aiFormFill/EntitiesTab.tsx @@ -0,0 +1,331 @@ +/** + * Entities tab — two-column layout: entity list grouped by type (left) + fields table (right). + */ +import { useState, useCallback } from 'react'; +import { + Stack, + Group, + Text, + Button, + TextInput, + ActionIcon, + NavLink, + Badge, + Table, + Menu, + Select, +} from '@mantine/core'; +import DeleteIcon from '@mui/icons-material/Delete'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import EditIcon from '@mui/icons-material/Edit'; +import ContentCopyIcon from '@mui/icons-material/ContentCopy'; +import AddIcon from '@mui/icons-material/Add'; +import { ALL_ENTITY_TYPES, ENTITY_TYPE_LABELS, type EntityType } from './entityTypes'; +import type { EntityStore } from './useEntityStore'; +import { + exportEntitiesToJson, + downloadJson, + importEntitiesFromJson, + importCsvToStore, +} from './entityImportExport'; +import FileUploadIcon from '@mui/icons-material/FileUpload'; +import FileDownloadIcon from '@mui/icons-material/FileDownload'; + +export function EntitiesTab({ store }: { store: EntityStore }) { + const [selectedId, setSelectedId] = useState(store.defaultEntityId); + const [newName, setNewName] = useState(''); + const [newType, setNewType] = useState('person'); + const [renamingId, setRenamingId] = useState(null); + const [renameValue, setRenameValue] = useState(''); + const [editingKey, setEditingKey] = useState(null); + const [editValue, setEditValue] = useState(''); + const [importMessage, setImportMessage] = useState(null); + + const selectedEntity = selectedId ? store.getEntity(selectedId) : undefined; + const entries = selectedEntity + ? Object.entries(selectedEntity.fields).filter(([k]) => !k.startsWith('_')) + : []; + + const handleCreate = useCallback(() => { + const name = newName.trim(); + if (name) { + const entity = store.createEntity(newType, name); + setSelectedId(entity.id); + setNewName(''); + } + }, [newName, newType, store]); + + const handleRename = useCallback(() => { + if (renamingId && renameValue.trim()) { + store.updateEntity(renamingId, { name: renameValue.trim() }); + } + setRenamingId(null); + setRenameValue(''); + }, [renamingId, renameValue, store]); + + const handleSaveEdit = useCallback( + (key: string) => { + if (selectedId) store.setField(selectedId, key, editValue); + setEditingKey(null); + setEditValue(''); + }, + [selectedId, editValue, store] + ); + + const typeSelectData = ALL_ENTITY_TYPES.map((t) => ({ + value: t, + label: ENTITY_TYPE_LABELS[t], + })); + + return ( +
+ {/* Left: entity list grouped by type */} +
+ + {/* Create new entity */} + + v && onEntityChange(v)} + placeholder="Select entity..." + searchable + styles={{ input: { fontSize: '0.8125rem' } }} + /> + + {/* Expand/collapse toggle */} + setExpanded(!expanded)}> + + {expanded ? : } + + {expanded ? 'Hide fields' : `Show fields (${Object.values(roleFieldNames).flat().length})`} + + + + + {/* Expandable field list per file */} + + + {role.fileIds.map((fileId) => { + const fields = fieldsByFile[fileId] || []; + const roleFields = new Set(roleFieldNames[fileId] || []); + const skipped = skippedFieldsByFile[fileId] || new Set(); + const cleanedLabels = cleanedLabelsByFile[fileId] || []; + const labelMap: Record = {}; + for (const cl of cleanedLabels) labelMap[cl.fieldName] = cl.label; + + const visibleFields = fields.filter( + (f) => roleFields.has(f.name) && !skipped.has(f.name) && !f.readOnly + ); + + const fileOverride = fileRoleOverrides[fileId]?.[role.roleLabel]; + + return ( + + {fileCount > 1 && ( + + {fileNames[fileId] || fileId} + {fileOverride && ( + override: {fileOverride} + )} + {fileCount > 1 && ( +