Set up document management for Stirling Engine (#6476)

# Description of Changes
Change Stirling Engine to support deleting documents automatically. This
happens both on user logout and after an amount of time specified by the
Java when ingesting a document (allowing for personal documents to have
short lifetimes but org documents to be left in the db with no expiry
date). Also sets up an [ACL
policy](https://en.wikipedia.org/wiki/Access-control_list) for the
documents so the database knows which users have access to which
documents. This is not fully implemented in the Java, so currently all
docs are treated as having a single owner, the uploader, but
theoretically when we need to support org storage, we shouldn't need to
change the db schema.

engine/src/stirling/documents/rag_capability.py

@@ -8,7 +8,7 @@ from pydantic_ai.toolsets import AbstractToolset
 
 from stirling.documents.service import DocumentService
 from stirling.documents.store import SearchResult
-from stirling.models import FileId
+from stirling.models import FileId, PrincipalId
 
 logger = logging.getLogger(__name__)
 
@@ -16,9 +16,13 @@ logger = logging.getLogger(__name__)
 class RagCapability:
     """Bundles RAG instructions and the ``search_knowledge`` toolset for agent injection.
 
-    Agents consume this as::
+    Agents construct one per run, scoped to the caller's principal set, as::
 
-        rag = runtime.rag_capability
+        rag = RagCapability(
+            documents=self.runtime.documents,
+            principals=current_principals(),
+            collections=[file.id for file in request.files],
+        )
         Agent(
             ...,
             instructions=rag.instructions,
@@ -26,20 +30,22 @@ class RagCapability:
         )
 
     When no collections are pinned, the instructions are generated dynamically at
-    run time so the agent sees the current list of collections in the store.
+    run time so the agent sees the collections the caller can read.
 
     Lifecycle: a ``RagCapability`` instance is intended to live for the duration of a
-    single agent run.
+    single agent run and binds to one caller's principal set.
     """
 
     def __init__(
         self,
         documents: DocumentService,
+        principals: list[PrincipalId],
         collections: list[FileId] | None = None,
         top_k: int = 5,
         max_searches: int = 5,
     ) -> None:
         self._documents = documents
+        self._principals = principals
         self._collections = collections
         self._top_k = top_k
         self._max_searches = max_searches
@@ -74,7 +80,7 @@ class RagCapability:
         )
 
     async def _dynamic_instructions(self) -> str:
-        collections = await self._documents.list_collections()
+        collections = await self._documents.list_collections(self._principals)
         if collections:
             names = ", ".join(collections)
             collection_desc = f"the following knowledge base collections: {names}"
@@ -115,12 +121,12 @@ class RagCapability:
         if self._collections:
             all_results = []
             for col in self._collections:
-                col_results = await self._documents.search(query, collection=col, top_k=k)
+                col_results = await self._documents.search(query, principals=self._principals, collection=col, top_k=k)
                 all_results.extend(col_results)
             all_results.sort(key=lambda r: r.score, reverse=True)
             results = all_results[:k]
         else:
-            results = await self._documents.search(query, top_k=k)
+            results = await self._documents.search(query, principals=self._principals, top_k=k)
 
         if not results:
             logger.info("[rag] search_knowledge query=%r -> 0 results", query)