Compare commits

..
Author SHA1 Message Date
Ludy87 98080a24d4 task frontend:fix 2026-05-13 21:07:45 +02:00
Ludy87 96e03dfe94 handle axios header typing 2026-05-13 21:00:34 +02:00
280 changed files with 5922 additions and 15954 deletions
+1 -1
View File
@@ -22,7 +22,7 @@ runs:
steps:
- name: Generate a GitHub App Token
id: generate-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
with:
app-id: ${{ inputs.app-id }}
private-key: ${{ inputs.private-key }}
-20
View File
@@ -85,26 +85,6 @@ jobs:
echo "to see what still needs fixing manually."
echo "============================================"
exit 1
- name: Remove frontend check comment on success
if: steps.frontend-check.outcome == 'success' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const marker = '<!-- frontend-check -->';
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
});
}
- name: Upload frontend build artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
+1 -2
View File
@@ -86,9 +86,8 @@ tasks:
fix:
desc: "Auto-fix lint + format"
cmds:
- task: format # Can auto-fix some things that `lint:fix` can't like line length violations
- task: lint:fix
- task: format # Ensure that after lint fixing that the code is still formatted correctly
- task: format
check:
desc: "Full engine quality gate"
+10 -10
View File
@@ -183,26 +183,26 @@ Use this pattern for desktop-specific or proprietary-specific features WITHOUT r
**Example - Desktop-specific footer:**
```typescript
// core/components/workbenchBar/WorkbenchBarFooterExtensions.tsx (stub)
interface WorkbenchBarFooterExtensionsProps {
// core/components/rightRail/RightRailFooterExtensions.tsx (stub)
interface RightRailFooterExtensionsProps {
className?: string;
}
export function WorkbenchBarFooterExtensions(_props: WorkbenchBarFooterExtensionsProps) {
export function RightRailFooterExtensions(_props: RightRailFooterExtensionsProps) {
return null; // Stub - does nothing in web builds
}
```
```tsx
// desktop/components/workbenchBar/WorkbenchBarFooterExtensions.tsx (real implementation)
// desktop/components/rightRail/RightRailFooterExtensions.tsx (real implementation)
import { Box } from '@mantine/core';
import { BackendHealthIndicator } from '@app/components/BackendHealthIndicator';
interface WorkbenchBarFooterExtensionsProps {
interface RightRailFooterExtensionsProps {
className?: string;
}
export function WorkbenchBarFooterExtensions({ className }: WorkbenchBarFooterExtensionsProps) {
export function RightRailFooterExtensions({ className }: RightRailFooterExtensionsProps) {
return (
<Box className={className}>
<BackendHealthIndicator />
@@ -212,15 +212,15 @@ export function WorkbenchBarFooterExtensions({ className }: WorkbenchBarFooterEx
```
```tsx
// core/components/shared/WorkbenchBar.tsx (usage - works in ALL builds)
import { WorkbenchBarFooterExtensions } from '@app/components/workbenchBar/WorkbenchBarFooterExtensions';
// core/components/shared/RightRail.tsx (usage - works in ALL builds)
import { RightRailFooterExtensions } from '@app/components/rightRail/RightRailFooterExtensions';
export function WorkbenchBar() {
export function RightRail() {
return (
<div>
{/* In web builds: renders nothing (stub returns null) */}
{/* In desktop builds: renders BackendHealthIndicator */}
<WorkbenchBarFooterExtensions className="workbench-bar-footer" />
<RightRailFooterExtensions className="right-rail-footer" />
</div>
);
}
-8
View File
@@ -51,12 +51,4 @@ dependencies {
api 'org.simplejavamail:outlook-module:8.12.6' // MSG file support
api 'jakarta.mail:jakarta.mail-api:2.1.5'
runtimeOnly 'org.eclipse.angus:angus-mail:2.0.5'
// Tabula table extraction — used by the shared PDF parser and directly by downstream modules.
// api-scoped so downstream modules (core, proprietary) retain it on their compile classpath.
api ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
}
@@ -1,73 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
* LineAlignmentTableParser}. The first parser returning a result above {@link
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
*/
@Service
@Primary
@RequiredArgsConstructor
@Slf4j
public class CompositeTableParser implements TableParser {
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
private final TabulaTableParser tabulaParser;
private final LineAlignmentTableParser lineAlignmentParser;
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
// Step 1: Tabula lattice mode (ruled/bordered tables).
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
if (!latticeResults.isEmpty()) {
log.debug(
"Page {}: using Tabula lattice ({} table(s))",
rawPage.pageNumber(),
latticeResults.size());
return latticeResults;
}
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
// parseStream is not on the TableParser interface — this intentionally couples to the
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
List<TableFragment> streamResults =
filterConfident(tabulaParser.parseStream(document, rawPage));
if (!streamResults.isEmpty()) {
log.debug(
"Page {}: using Tabula stream ({} table(s))",
rawPage.pageNumber(),
streamResults.size());
return streamResults;
}
// Step 3: Geometry-based line-alignment fallback.
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
if (!lineResults.isEmpty()) {
log.debug(
"Page {}: using LineAlignment ({} table(s))",
rawPage.pageNumber(),
lineResults.size());
return lineResults;
}
return List.of();
}
private List<TableFragment> filterConfident(List<TableFragment> tables) {
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
}
}
@@ -1,528 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Fallback {@link TableParser} for borderless financial tables using text geometry.
*
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
* on column consistency and anchor density (confidence ceiling 0.85).
*/
@Service
@Slf4j
public class LineAlignmentTableParser implements TableParser {
/** Width in points of each column position bucket. */
static final float COLUMN_BUCKET_PT = 5f;
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
private static final int COLUMN_MATCH_BUCKETS = 2;
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
private static final float MAX_GAP_FACTOR = 2.5f;
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
static final int MIN_TABLE_ROWS = 3;
/** Minimum confirmed column positions to form a valid table. */
static final int MIN_COLUMNS = 2;
/**
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
* rows).
*/
private static final double COLUMN_MIN_FREQUENCY = 0.40;
/**
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
* percent, nil dashes.
*/
private static final Pattern NUMERIC =
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
/**
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
* column-gap logic).
*/
static final float ROW_MERGE_TOLERANCE_PT = 2f;
// ── public API ───────────────────────────────────────────────────────────────────────────────
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
List<RawLine> lines = rawPage.lines();
if (lines.size() < MIN_TABLE_ROWS) return List.of();
float modalSpacing = computeModalSpacing(lines);
List<TokenizedLine> tokenized =
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
List<Float> columnGrid = buildColumnGrid(anchors);
if (columnGrid.size() < MIN_COLUMNS) {
log.debug(
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
rawPage.pageNumber(),
MIN_COLUMNS);
return List.of();
}
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
List<TableFragment> results = new ArrayList<>();
for (int i = 0; i < groups.size(); i++) {
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
.ifPresent(results::add);
}
log.debug(
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
rawPage.pageNumber(),
results.size(),
anchors.size(),
columnGrid.size());
return results;
}
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
/**
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
* split by LineBuilder.
*/
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
if (tokenized.size() < 2) return tokenized;
List<TokenizedLine> result = new ArrayList<>();
int i = 0;
while (i < tokenized.size()) {
float baseY = tokenized.get(i).line().bounds().y();
int j = i + 1;
while (j < tokenized.size()
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
<= ROW_MERGE_TOLERANCE_PT) {
j++;
}
if (j == i + 1) {
result.add(tokenized.get(i));
} else {
result.add(mergeGroup(tokenized.subList(i, j)));
}
i = j;
}
return result;
}
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
List<TextFragment> mergedFragments =
group.stream()
.flatMap(tl -> tl.line().fragments().stream())
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds mergedBounds =
group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(group.get(0).line().bounds());
RawLine mergedLine =
new RawLine(
group.get(0).line().lineId(),
mergedFragments,
mergedBounds,
group.get(0).line().pageNumber());
return tokenize(mergedLine);
}
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
/**
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
* fragment.
*/
TokenizedLine tokenize(RawLine line) {
List<LineToken> tokens = new ArrayList<>();
for (TextFragment frag : line.fragments()) {
tokens.addAll(tokensFromFragment(frag));
}
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
return new TokenizedLine(line, tokens, numeric);
}
private List<LineToken> tokensFromFragment(TextFragment frag) {
String raw = frag.text();
if (raw == null || raw.isBlank()) return List.of();
float fragX = frag.bounds().x();
float fragWidth = frag.bounds().width();
int rawLen = raw.length();
List<LineToken> result = new ArrayList<>();
int offset = 0;
for (String part : raw.split("\\s+")) {
if (part.isEmpty()) {
offset++;
continue;
}
int idx = raw.indexOf(part, offset);
if (idx < 0) idx = offset;
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
float tokenRight =
rawLen > 0
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
: fragX + fragWidth;
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
offset = idx + part.length();
}
return result;
}
// ── column grid ──────────────────────────────────────────────────────────────────────────────
/**
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
*/
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
// bucket → set of line indices that contributed a numeric token to that bucket
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
for (int i = 0; i < anchors.size(); i++) {
for (LineToken t : anchors.get(i).numeric()) {
int bucket = bucket(t.right());
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
}
}
int minHits =
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
// Confirmed buckets → average right-edge for that bucket
TreeMap<Integer, Float> confirmed = new TreeMap<>();
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
// Count distinct lines
long distinctLines = entry.getValue().stream().distinct().count();
if (distinctLines >= minHits) {
double avg =
entry.getValue().stream()
.distinct() // weight each line equally regardless of token count
.mapToDouble(
lineIdx ->
avgRightEdgeForBucket(
anchors, lineIdx, entry.getKey()))
.average()
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
confirmed.put(entry.getKey(), (float) avg);
}
}
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
}
/**
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
*/
private double avgRightEdgeForBucket(
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
return anchors.get(lineIdx).numeric().stream()
.filter(t -> bucket(t.right()) == targetBucket)
.mapToDouble(LineToken::right)
.average()
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
}
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
/**
* Groups anchor lines into table candidates, including adjacent label rows; a gap &gt;
* MAX_GAP_FACTOR × modal spacing splits groups.
*/
private List<List<TokenizedLine>> groupRows(
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
List<List<TokenizedLine>> groups = new ArrayList<>();
List<TokenizedLine> current = new ArrayList<>();
for (int i = 0; i < all.size(); i++) {
TokenizedLine tl = all.get(i);
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
if (current.isEmpty()) {
if (fits) current.add(tl);
continue;
}
float gap =
tl.line().bounds().y()
- current.get(current.size() - 1).line().bounds().bottom();
if (gap > maxGap) {
groups.add(current);
current = new ArrayList<>();
if (fits) current.add(tl);
continue;
}
if (fits) {
current.add(tl);
} else if (!tl.line().text().isBlank()) {
// Include non-anchor lines (labels) only if they have text and are within
// proximity.
current.add(tl);
}
}
if (!current.isEmpty()) groups.add(current);
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
}
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
>= MIN_TABLE_ROWS;
}
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long matches =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) matches / tl.numeric().size() >= 0.60;
}
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long hits =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) hits / tl.numeric().size() < 0.60;
}
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
private Optional<TableFragment> buildFragment(
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
List<String> warnings = new ArrayList<>();
List<List<String>> rawRows = new ArrayList<>();
List<TableRow> rows = new ArrayList<>();
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
TokenizedLine tl = group.get(rowIdx);
List<String> rawRow = buildRawRow(tl, columnGrid);
rawRows.add(Collections.unmodifiableList(rawRow));
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
}
// Column count = 1 label column + confirmed numeric columns
int colCount = columnGrid.size() + 1;
Bounds bounds = computeGroupBounds(group);
float confidence = computeConfidence(group, columnGrid, warnings);
return Optional.of(
new TableFragment(
"tbl-la-p" + pageNumber + "-" + tableIndex,
pageNumber,
bounds,
List.of(),
Collections.unmodifiableList(rows),
Collections.unmodifiableList(rawRows),
colCount,
confidence,
Collections.unmodifiableList(warnings),
null));
}
/**
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
*/
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
String[] cells = new String[columnGrid.size() + 1];
Arrays.fill(cells, "");
// Separate label tokens (those not landing in any confirmed column) from column tokens.
List<String> labelParts = new ArrayList<>();
for (LineToken token : tl.all()) {
int col = nearestColumnIndex(token.right(), columnGrid);
if (col >= 0 && token.numeric()) {
int cellIdx = col + 1;
cells[cellIdx] =
cells[cellIdx].isEmpty()
? token.text()
: cells[cellIdx] + " " + token.text();
} else {
labelParts.add(token.text());
}
}
cells[0] = String.join(" ", labelParts).trim();
return Arrays.asList(cells);
}
private TableRow buildTableRow(
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
List<TableCell> cells = new ArrayList<>(rawRow.size());
// Label cell: use the line's full bounds as an approximation.
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
for (int col = 0; col < columnGrid.size(); col++) {
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
float right = columnGrid.get(col);
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
Bounds cellBounds =
new Bounds(
left,
tl.line().bounds().y(),
right - left,
tl.line().bounds().height());
cells.add(TableCell.of(col + 1, text, cellBounds));
}
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
}
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
/**
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
* &gt;30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
*/
private float computeConfidence(
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
float score = 0.70f;
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
long totalRows = group.size();
// More columns
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
score += extraCols * 0.05f;
// More anchor rows
if (anchorCount >= 5) score += 0.05f;
if (anchorCount >= 8) score += 0.05f;
// Inconsistent column matching
long inconsistent =
group.stream()
.filter(TokenizedLine::isAnchor)
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
.count();
if (inconsistent > anchorCount * 0.30) {
score -= 0.15f;
warnings.add(
"Column match inconsistent on "
+ inconsistent
+ "/"
+ anchorCount
+ " anchor rows");
}
// Label-heavy
long nonAnchor = totalRows - anchorCount;
if (nonAnchor > anchorCount) {
score -= 0.10f;
warnings.add(
"Non-anchor rows ("
+ nonAnchor
+ ") outnumber anchor rows ("
+ anchorCount
+ ")");
}
return Math.max(0f, Math.min(0.85f, score));
}
// ── utility ──────────────────────────────────────────────────────────────────────────────────
/**
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
* #COLUMN_MATCH_BUCKETS} buckets.
*/
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
int nearest = -1;
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
for (int i = 0; i < grid.size(); i++) {
float dist = Math.abs(rightEdge - grid.get(i));
if (dist < minDist) {
minDist = dist;
nearest = i;
}
}
return nearest;
}
private Bounds computeGroupBounds(List<TokenizedLine> group) {
return group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
}
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
private float computeModalSpacing(List<RawLine> lines) {
if (lines.size() < 2) return 0f;
Map<Float, Long> freq = new HashMap<>();
for (int i = 1; i < lines.size(); i++) {
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
}
return freq.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse(0f);
}
private static int bucket(float x) {
return Math.round(x / COLUMN_BUCKET_PT);
}
// ── private data types ───────────────────────────────────────────────────────────────────────
/** A word-level token with an approximate right-edge x-position. */
record LineToken(String text, float x, float right, boolean numeric) {}
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
boolean isAnchor() {
return numeric.size() >= 2;
}
}
}
@@ -1,139 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
*
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
*/
@Service
@Slf4j
public class LineBuilder {
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
private static final float MIN_BASELINE_TOLERANCE = 2f;
/**
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
* two-column gutters.
*/
static final float COLUMN_GAP_RATIO = 0.10f;
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
static final float COLUMN_GAP_MIN_PT = 40f;
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
if (fragments.isEmpty()) return List.of();
float effectiveWidth = inferEffectiveWidth(fragments);
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
log.debug(
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
pageNumber,
effectiveWidth,
columnGapThreshold);
// Sort top-to-bottom first, then left-to-right within the same baseline band.
List<TextFragment> sorted =
fragments.stream()
.sorted(
Comparator.comparingDouble(TextFragment::baseline)
.thenComparingDouble(f -> f.bounds().x()))
.toList();
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
List<RawLine> lines = new ArrayList<>(groups.size());
for (int i = 0; i < groups.size(); i++) {
List<TextFragment> group =
groups.get(i).stream()
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds lineBounds =
group.stream()
.map(TextFragment::bounds)
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
}
return lines;
}
private List<List<TextFragment>> groupByBaseline(
List<TextFragment> sorted, float columnGapThreshold) {
List<List<TextFragment>> groups = new ArrayList<>();
List<TextFragment> current = new ArrayList<>();
float currentBaseline = Float.NaN;
for (TextFragment fragment : sorted) {
if (current.isEmpty()) {
current.add(fragment);
currentBaseline = fragment.baseline();
continue;
}
float maxFontSize =
Math.max(
fragment.fontSize(),
(float)
current.stream()
.mapToDouble(TextFragment::fontSize)
.max()
.orElse(0));
float tolerance =
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
if (sameBaseline && !columnGap) {
current.add(fragment);
// Anchor to the weighted mean baseline so long lines stay stable.
currentBaseline =
(currentBaseline * (current.size() - 1) + fragment.baseline())
/ current.size();
} else {
groups.add(current);
current = new ArrayList<>();
current.add(fragment);
currentBaseline = fragment.baseline();
}
}
if (!current.isEmpty()) groups.add(current);
return groups;
}
/**
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
* threshold}.
*/
private static boolean hasColumnGap(
TextFragment next, List<TextFragment> group, float threshold) {
float lastRight = group.get(group.size() - 1).bounds().right();
return next.bounds().x() - lastRight > threshold;
}
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
private static float inferEffectiveWidth(List<TextFragment> fragments) {
double maxRight =
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
return (float) maxRight * 1.10f;
}
}
@@ -1,79 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
* {@link PDDocument} lifecycle.
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class PdfIngester {
private final LineBuilder lineBuilder;
private final TableParser tableParser;
public List<ParsedPage> parse(PDDocument document) throws IOException {
return parse(document, document.getNumberOfPages());
}
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
List<ParsedPage> pages = new ArrayList<>(pageCount);
long fragmentsMs = 0;
long tablesMs = 0;
long t0 = System.currentTimeMillis();
for (int p = 1; p <= pageCount; p++) {
long ft = System.currentTimeMillis();
List<TextFragment> fragments = extractFragments(document, p);
fragmentsMs += System.currentTimeMillis() - ft;
PDPage page = document.getPage(p - 1);
PDRectangle mediaBox = page.getMediaBox();
List<RawLine> lines = lineBuilder.build(fragments, p);
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
long tt = System.currentTimeMillis();
List<TableFragment> tables = tableParser.parse(document, rawPage);
tablesMs += System.currentTimeMillis() - tt;
log.debug(
"Page {}: {} fragments → {} lines, {} table(s)",
p,
fragments.size(),
lines.size(),
tables.size());
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
}
log.info(
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
pageCount,
System.currentTimeMillis() - t0,
fragmentsMs,
tablesMs);
return pages;
}
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
throws IOException {
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
stripper.getText(document);
return stripper.getFragments();
}
}
@@ -1,148 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
/**
* All PDF parser model types and the table-parser contract in one place.
*
* <p>Import as {@code import static stirling.software.SPDF.pdf.parser.PdfModels.*;} to use all
* nested types without qualification.
*/
public final class PdfModels {
private PdfModels() {}
// ── Geometry ──────────────────────────────────────────────────────────────
public record Bounds(float x, float y, float width, float height) {
public float right() {
return x + width;
}
public float bottom() {
return y + height;
}
public static Bounds merge(Bounds a, Bounds b) {
float x = Math.min(a.x, b.x);
float y = Math.min(a.y, b.y);
return new Bounds(
x, y, Math.max(a.right(), b.right()) - x, Math.max(a.bottom(), b.bottom()) - y);
}
}
// ── Text fragments and lines ──────────────────────────────────────────────
/**
* A contiguous run of text from a single PDF content-stream string operation. {@code baseline}
* is preserved separately from bounds for line-grouping — characters of different sizes on the
* same visual line share a baseline but differ in top Y.
*/
public record TextFragment(
String fragmentId,
String text,
Bounds bounds,
float baseline,
float fontSize,
String fontName,
boolean bold) {}
public record RawLine(
String lineId, List<TextFragment> fragments, Bounds bounds, int pageNumber) {
public String text() {
if (fragments.isEmpty()) return "";
StringBuilder sb = new StringBuilder();
TextFragment prev = null;
for (TextFragment f : fragments) {
if (prev != null) {
float gap = f.bounds().x() - prev.bounds().right();
float avgCharWidth = prev.bounds().width() / Math.max(prev.text().length(), 1);
if (gap > avgCharWidth * 0.5f) sb.append(' ');
}
sb.append(f.text());
prev = f;
}
return sb.toString();
}
public float dominantFontSize() {
return fragments.stream()
.collect(Collectors.groupingBy(TextFragment::fontSize, Collectors.counting()))
.entrySet()
.stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse(0f);
}
public boolean hasBold() {
return fragments.stream().anyMatch(TextFragment::bold);
}
}
public record RawPage(int pageNumber, float widthPt, float heightPt, List<RawLine> lines) {}
// ── Table model ───────────────────────────────────────────────────────────
/**
* A single cell within a {@link TableRow}. {@code colSpan} and {@code rowSpan} are always 1 in
* v1 — span detection is deferred but the fields keep the schema stable.
*/
public record TableCell(int colIndex, String text, Bounds bounds, int colSpan, int rowSpan) {
public static TableCell of(int colIndex, String text, Bounds bounds) {
return new TableCell(colIndex, text, bounds, 1, 1);
}
}
public record TableRow(int rowIndex, List<TableCell> cells) {}
/**
* A table as it appears on a single page. {@code headers} is empty in v1 — all rows are in
* {@code rows}. {@code rawRows} preserves exact Tabula text output for diagnostics. {@code
* confidence} is a heuristic score in [0.0, 1.0]. {@code continuedFromPage} is null in v1.
*/
public record TableFragment(
String tableId,
int pageNumber,
Bounds bounds,
List<TableRow> headers,
List<TableRow> rows,
List<List<String>> rawRows,
int columnCount,
float confidence,
List<String> warnings,
Integer continuedFromPage) {}
// ── Page output ───────────────────────────────────────────────────────────
public record ParsedPage(
int pageNumber,
float widthPt,
float heightPt,
List<TableFragment> tables,
List<RawLine> layoutLines) {}
// ── Parser contract ───────────────────────────────────────────────────────
/**
* Extracts tables from a single page of a PDF. The caller owns the document lifecycle;
* implementations must not close it.
*/
public interface TableParser {
/**
* @param document open PDF; must not be closed by the implementation
* @param rawPage page metadata and lines for the page to process
* @return zero or more table fragments found on the page, never null
*/
List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException;
}
}
@@ -1,256 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.RectangularTextContainer;
import technology.tabula.Table;
import technology.tabula.extractors.BasicExtractionAlgorithm;
import technology.tabula.extractors.ExtractionAlgorithm;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
/**
* Primary {@link TableParser} implementation using Tabula's lattice-mode extraction.
*
* <h3>Algorithm</h3>
*
* Uses {@link SpreadsheetExtractionAlgorithm} (lattice mode), which detects tables from ruled lines
* (horizontal and vertical PDF path operators). This is reliable for tables with visible borders.
* Borderless or whitespace-delimited tables will not be detected — that requires stream mode, which
* is deferred to a later iteration.
*
* <h3>Coordinate system</h3>
*
* Tabula normalises page coordinates so that (0,0) is the top-left corner and Y increases downward,
* matching the {@link Bounds} convention used throughout this parser. No coordinate transformation
* is needed when mapping Tabula cell bounds to {@link Bounds}.
*
* <h3>Known limitations</h3>
*
* <ul>
* <li>Borderless tables are not detected (use stream mode, deferred).
* <li>Colspan and rowspan are not detected; all cells report colSpan=1, rowSpan=1.
* <li>Header rows are not identified; all rows appear in {@code rows}, headers is empty.
* <li>Cross-page table linking is not performed; each page is independent.
* <li>Rotated tables (90°/270° pages) may produce incorrect bounds.
* </ul>
*/
@Service
@Slf4j
public class TabulaTableParser implements TableParser {
/** Lattice mode — reliable for tables with visible ruled borders. */
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
return parseWithAlgorithm(
document, rawPage, new SpreadsheetExtractionAlgorithm(), "lattice");
}
/**
* Convenience overload for callers that only have a page number, not a full {@link RawPage}.
*/
public List<TableFragment> parse(PDDocument document, int pageNumber) throws IOException {
return parse(document, new RawPage(pageNumber, 0f, 0f, List.of()));
}
/** Stream mode — whitespace-based column detection for borderless tables. */
public List<TableFragment> parseStream(PDDocument document, RawPage rawPage)
throws IOException {
return parseWithAlgorithm(document, rawPage, new BasicExtractionAlgorithm(), "stream");
}
private List<TableFragment> parseWithAlgorithm(
PDDocument document, RawPage rawPage, ExtractionAlgorithm algorithm, String modeName)
throws IOException {
int pageNumber = rawPage.pageNumber();
List<Table> tabulaTables;
try {
// Do NOT use try-with-resources: ObjectExtractor.close() closes the underlying
// PDDocument, which we don't own. The extractor holds no resources of its own.
ObjectExtractor extractor = new ObjectExtractor(document);
Page page = extractor.extract(pageNumber);
tabulaTables = new ArrayList<>(algorithm.extract(page));
} catch (Exception e) {
log.warn(
"Tabula {} extraction failed on page {}: {}",
modeName,
pageNumber,
e.getMessage());
return List.of();
}
if (tabulaTables.isEmpty()) {
log.debug("Page {}: no tables detected by Tabula ({})", pageNumber, modeName);
return List.of();
}
log.debug(
"Page {}: Tabula ({}) detected {} table(s)",
pageNumber,
modeName,
tabulaTables.size());
List<TableFragment> fragments = new ArrayList<>(tabulaTables.size());
for (int i = 0; i < tabulaTables.size(); i++) {
fragments.add(toFragment(tabulaTables.get(i), pageNumber, i));
}
return fragments;
}
// ── private helpers ──────────────────────────────────────────────────────────────────────────
private TableFragment toFragment(Table table, int pageNumber, int tableIndex) {
List<List<RectangularTextContainer>> tabulaRows = table.getRows();
List<String> warnings = new ArrayList<>();
List<List<String>> rawRows = buildRawRows(tabulaRows);
int colCount = inferColumnCount(rawRows, warnings);
List<TableRow> rows = buildRows(tabulaRows, colCount, warnings);
float confidence = computeConfidence(rawRows, colCount, warnings);
Bounds bounds = tableBounds(table);
String tableId = "tbl-p" + pageNumber + "-" + tableIndex;
if (!warnings.isEmpty()) {
log.warn("Page {}, table {}: {}", pageNumber, tableIndex, warnings);
}
return new TableFragment(
tableId,
pageNumber,
bounds,
List.of(), // headers: deferred to v2
rows,
rawRows,
colCount,
confidence,
Collections.unmodifiableList(warnings),
null); // continuedFromPage: deferred to v2
}
private List<List<String>> buildRawRows(List<List<RectangularTextContainer>> tabulaRows) {
List<List<String>> rawRows = new ArrayList<>(tabulaRows.size());
for (List<RectangularTextContainer> tabulaRow : tabulaRows) {
List<String> cells = new ArrayList<>(tabulaRow.size());
for (RectangularTextContainer cell : tabulaRow) {
cells.add(normaliseText(cell.getText()));
}
rawRows.add(Collections.unmodifiableList(cells));
}
return rawRows;
}
private List<TableRow> buildRows(
List<List<RectangularTextContainer>> tabulaRows, int colCount, List<String> warnings) {
List<TableRow> rows = new ArrayList<>(tabulaRows.size());
for (int rowIdx = 0; rowIdx < tabulaRows.size(); rowIdx++) {
List<RectangularTextContainer> tabulaRow = tabulaRows.get(rowIdx);
List<TableCell> cells = new ArrayList<>(tabulaRow.size());
for (int colIdx = 0; colIdx < tabulaRow.size(); colIdx++) {
RectangularTextContainer c = tabulaRow.get(colIdx);
Bounds cellBounds =
new Bounds(
(float) c.getX(),
(float) c.getY(),
(float) c.getWidth(),
(float) c.getHeight());
cells.add(TableCell.of(colIdx, normaliseText(c.getText()), cellBounds));
}
if (tabulaRow.size() != colCount) {
warnings.add(
"Row "
+ rowIdx
+ " has "
+ tabulaRow.size()
+ " cells; expected "
+ colCount);
}
rows.add(new TableRow(rowIdx, Collections.unmodifiableList(cells)));
}
return rows;
}
/**
* The canonical column count for a table is the size of the widest row. Tabula can produce
* uneven rows when a cell's ruling lines are partially missing.
*/
private int inferColumnCount(List<List<String>> rawRows, List<String> warnings) {
if (rawRows.isEmpty()) return 0;
int max = rawRows.stream().mapToInt(List::size).max().orElse(0);
int mode =
rawRows.stream()
.collect(
java.util.stream.Collectors.groupingBy(
List::size, java.util.stream.Collectors.counting()))
.entrySet()
.stream()
.max(java.util.Map.Entry.comparingByValue())
.map(java.util.Map.Entry::getKey)
.orElse(0);
if (max != mode) {
warnings.add("Inconsistent column count: modal=" + mode + " max=" + max);
}
return mode > 0 ? mode : max;
}
/**
* Heuristic confidence score in [0.0, 1.0].
*
* <ul>
* <li>Starts at 1.0.
* <li>-0.3 if only one column (single-column tables are usually not real tables).
* <li>-0.1 per row with an inconsistent column count, capped at -0.4.
* <li>-0.3 if the empty-cell ratio across all cells exceeds 80%.
* </ul>
*/
private float computeConfidence(
List<List<String>> rawRows, int colCount, List<String> warnings) {
if (rawRows.isEmpty() || colCount == 0) return 0f;
float score = 1.0f;
if (colCount == 1) score -= 0.3f;
long inconsistentRows = rawRows.stream().filter(r -> r.size() != colCount).count();
score -= Math.min(inconsistentRows * 0.1f, 0.4f);
long totalCells = rawRows.stream().mapToLong(List::size).sum();
long emptyCells =
rawRows.stream().flatMap(Collection::stream).filter(String::isBlank).count();
if (totalCells > 0 && (float) emptyCells / totalCells > 0.8f) {
score -= 0.3f;
}
return Math.max(0f, Math.min(1f, score));
}
private Bounds tableBounds(Table table) {
return new Bounds(
(float) table.getX(),
(float) table.getY(),
(float) table.getWidth(),
(float) table.getHeight());
}
private String normaliseText(String raw) {
if (raw == null) return "";
// Tabula wraps multi-line cell content with \r\n — collapse to a single space.
return raw.replace("\r\n", " ").replace("\n", " ").replace("\r", " ").trim();
}
}
@@ -1,113 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
/**
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
*
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
*/
class WordExtractingStripper extends PDFTextStripper {
private final int targetPage;
private final List<TextFragment> fragments = new ArrayList<>();
private int fragmentIndex = 0;
WordExtractingStripper(int pageNumber) throws IOException {
this.targetPage = pageNumber;
setStartPage(pageNumber);
setEndPage(pageNumber);
setSortByPosition(true);
}
@Override
protected void startPage(PDPage page) throws IOException {
super.startPage(page);
fragments.clear();
fragmentIndex = 0;
}
@Override
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
if (text == null || text.isBlank()) return;
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
// number as its own string operation, so this is the common case).
if (text.indexOf(' ') < 0) {
emitFragment(text, textPositions);
return;
}
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
if (textPositions.size() != text.length()) {
emitFragment(text, textPositions);
return;
}
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
int start = 0;
for (int i = 0; i <= text.length(); i++) {
if (i == text.length() || text.charAt(i) == ' ') {
if (start < i) {
emitFragment(text.substring(start, i), textPositions.subList(start, i));
}
start = i + 1;
}
}
}
private void emitFragment(String text, List<TextPosition> positions) {
if (positions.isEmpty()) return;
float minX = Float.MAX_VALUE;
float minY = Float.MAX_VALUE;
float maxRight = -Float.MAX_VALUE;
float maxBaseline = -Float.MAX_VALUE;
TextPosition first = null;
for (TextPosition tp : positions) {
if (tp == null) continue;
if (first == null) first = tp;
float x = tp.getX();
// getY() is the baseline; top of character = getY() - getHeight().
float top = tp.getY() - tp.getHeight();
float right = x + tp.getWidth();
float baseline = tp.getY();
minX = Math.min(minX, x);
minY = Math.min(minY, top);
maxRight = Math.max(maxRight, right);
maxBaseline = Math.max(maxBaseline, baseline);
}
if (first == null) return;
PDFont font = first.getFont();
String fontName = font != null ? font.getName() : "";
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
float fontSize = first.getHeight();
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
}
List<TextFragment> getFragments() {
return Collections.unmodifiableList(fragments);
}
}
@@ -14,6 +14,7 @@ import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@EqualsAndHashCode
@Schema(description = "PDF file input - either upload a file or provide a server-side file ID")
public class PDFFile {
@Schema(
@@ -2289,85 +2289,6 @@ public class FormUtils {
acroForm.getFields().add(field);
}
/** Drops AcroForm fields whose widgets are no longer on any page of {@code document}. */
public void pruneOrphanedFormFields(PDDocument document) {
if (document == null) {
return;
}
PDDocumentCatalog catalog = document.getDocumentCatalog();
if (catalog == null) {
return;
}
PDAcroForm form = catalog.getAcroForm(null);
if (form == null) {
return;
}
List<PDField> fields = form.getFields();
if (fields.isEmpty()) {
return;
}
Set<COSDictionary> liveWidgets = collectLiveWidgetDictionaries(document);
List<PDField> kept = pruneFieldList(fields, liveWidgets);
if (kept.isEmpty()) {
catalog.setAcroForm(null);
} else if (kept.size() != fields.size()) {
form.setFields(kept);
}
}
private Set<COSDictionary> collectLiveWidgetDictionaries(PDDocument document) {
Set<COSDictionary> live = new HashSet<>();
int pageCount = document.getNumberOfPages();
for (int i = 0; i < pageCount; i++) {
try {
for (PDAnnotation annotation : document.getPage(i).getAnnotations()) {
if (annotation instanceof PDAnnotationWidget) {
live.add(annotation.getCOSObject());
}
}
} catch (IOException e) {
log.debug("Failed reading page {} annotations: {}", i, e.getMessage());
}
}
return live;
}
private List<PDField> pruneFieldList(List<PDField> fields, Set<COSDictionary> liveWidgets) {
List<PDField> kept = new ArrayList<>(fields.size());
for (PDField field : fields) {
if (field instanceof PDNonTerminalField nonTerminal) {
List<PDField> children = nonTerminal.getChildren();
List<PDField> remaining = pruneFieldList(children, liveWidgets);
if (remaining.isEmpty()) {
continue;
}
if (remaining.size() != children.size()) {
nonTerminal.setChildren(remaining);
}
kept.add(nonTerminal);
} else if (field instanceof PDTerminalField terminal) {
List<PDAnnotationWidget> widgets = terminal.getWidgets();
List<PDAnnotationWidget> liveOnes = new ArrayList<>(widgets.size());
for (PDAnnotationWidget widget : widgets) {
if (liveWidgets.contains(widget.getCOSObject())) {
liveOnes.add(widget);
}
}
if (liveOnes.isEmpty()) {
continue;
}
if (liveOnes.size() != widgets.size()) {
terminal.setWidgets(liveOnes);
}
kept.add(terminal);
} else {
kept.add(field);
}
}
return kept;
}
// Delegation methods to GeneralFormCopyUtils for form field transformation
public boolean hasAnyRotatedPage(PDDocument document) {
return stirling.software.common.util.GeneralFormCopyUtils.hasAnyRotatedPage(document);
@@ -1,153 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static org.assertj.core.api.Assertions.assertThat;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.List;
import org.junit.jupiter.api.Test;
/**
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
* column-grid construction.
*/
class LineAlignmentTableParserTest {
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
@Test
void mergeCoincidentLines_singleLine_unchanged() {
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
}
@Test
void mergeCoincidentLines_distinctYLines_unchanged() {
// Two lines at different y positions — must NOT be merged.
var lines =
List.of(
tokenized(rawLine(10f, 100f, "Revenue")),
tokenized(rawLine(10f, 115f, "Cost")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
}
@Test
void mergeCoincidentLines_sameY_merged() {
// Simulates a financial-table row split by LineBuilder at the column gap:
// label fragment at x=72 → "Revenue"
// value fragment at x=350 → "1,234"
// Both have y=100. After merge they should form one TokenizedLine.
var label = rawLine(72f, 100f, "Revenue");
var value = rawLine(350f, 100f, "1,234");
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
assertThat(merged).hasSize(1);
// The merged line should contain tokens from both halves.
var tokens = merged.get(0).all();
assertThat(tokens.stream().map(t -> t.text()).toList())
.containsExactlyInAnyOrder("Revenue", "1,234");
}
@Test
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
var bounds = merged.get(0).line().bounds();
assertThat(bounds.x()).isEqualTo(72f);
assertThat(bounds.right()).isEqualTo(380f);
}
@Test
void mergeCoincidentLines_withinTolerance_merged() {
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 101.5f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(1);
}
@Test
void mergeCoincidentLines_beyondTolerance_notMerged() {
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 103.0f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_threeCoincident_allMerged() {
// Three fragments at the same y (e.g. wide financial table with two value columns).
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(300f, 100f, "1,234");
var c = rawLine(400f, 100f, "5,678");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).all()).hasSize(3);
}
@Test
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
// After merging, the combined line should be an anchor (≥2 numeric tokens).
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
// Merged → anchor with at least 2 numerics.
var label = rawLine(72f, 100f, "Revenue");
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).isAnchor()).isTrue();
}
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
private static RawLine rawLine(float x, float y, String text) {
float width = text.length() * 6f; // ~6pt per char — rough but consistent
float height = 12f;
Bounds bounds = new Bounds(x, y, width, height);
TextFragment fragment =
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
return new RawLine("ln-test", List.of(fragment), bounds, 1);
}
/**
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
* Used to simulate a values-only line with multiple numeric tokens.
*/
private static RawLine rawLineMultiWord(
float x, float y, String word1, float w1, String word2, float w2) {
float height = 12f;
Bounds b1 = new Bounds(x, y, w1, height);
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
}
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
return parser.tokenize(line);
}
}
@@ -1,229 +0,0 @@
package stirling.software.common.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationWidget;
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
import org.apache.pdfbox.pdmodel.interactive.form.PDNonTerminalField;
import org.apache.pdfbox.pdmodel.interactive.form.PDTextField;
import org.junit.jupiter.api.Test;
class FormUtilsPruneOrphanedFieldsTest {
@Test
void noAcroFormIsNoOp() throws IOException {
try (PDDocument document = new PDDocument()) {
document.addPage(new PDPage(PDRectangle.A4));
FormUtils.pruneOrphanedFormFields(document);
assertNull(document.getDocumentCatalog().getAcroForm(null));
}
}
@Test
void dropsFieldsWhoseWidgetsAreAllOnRemovedPages() throws IOException {
byte[] pdfBytes = buildPdfWithFieldPerPage(3);
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(2);
document.removePage(0);
FormUtils.pruneOrphanedFormFields(document);
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
List<String> remainingNames = new ArrayList<>();
for (PDField field : form.getFields()) {
remainingNames.add(field.getPartialName());
}
assertEquals(List.of("field_1"), remainingNames);
}
}
@Test
void dropsAcroFormEntirelyWhenNoFieldsSurvive() throws IOException {
byte[] pdfBytes = buildPdfWithFieldPerPage(2);
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(1);
document.removePage(0);
document.addPage(new PDPage(PDRectangle.A4));
FormUtils.pruneOrphanedFormFields(document);
assertNull(document.getDocumentCatalog().getAcroForm(null));
}
}
@Test
void keepsLiveWidgetsAndDropsOrphanWidgetsFromMultiWidgetField() throws IOException {
byte[] pdfBytes = buildPdfWithMultiWidgetField();
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(0);
FormUtils.pruneOrphanedFormFields(document);
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
assertEquals(1, form.getFields().size());
PDField field = form.getFields().get(0);
assertEquals("multi", field.getPartialName());
assertEquals(2, field.getWidgets().size(), "two widgets remain after one is dropped");
}
}
@Test
void survivesRoundTripWithoutOrphanPagesInOutput() throws IOException {
byte[] pdfBytes = buildPdfWithFieldPerPage(3);
byte[] writtenBytes;
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(2);
document.removePage(1);
FormUtils.pruneOrphanedFormFields(document);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
writtenBytes = out.toByteArray();
}
}
try (PDDocument reloaded = Loader.loadPDF(writtenBytes)) {
assertEquals(1, reloaded.getNumberOfPages());
PDAcroForm form = reloaded.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
assertEquals(1, form.getFields().size());
assertEquals("field_0", form.getFields().get(0).getPartialName());
}
}
@Test
void prunesNestedNonTerminalFields() throws IOException {
byte[] pdfBytes = buildPdfWithNestedFields();
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
document.removePage(1);
FormUtils.pruneOrphanedFormFields(document);
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
assertNotNull(form);
assertEquals(1, form.getFields().size());
PDField group = form.getFields().get(0);
assertEquals("group", group.getPartialName());
assertTrue(group instanceof PDNonTerminalField);
PDNonTerminalField nonTerminal = (PDNonTerminalField) group;
assertEquals(1, nonTerminal.getChildren().size());
assertEquals("kept", nonTerminal.getChildren().get(0).getPartialName());
}
}
private static byte[] buildPdfWithFieldPerPage(int pageCount) throws IOException {
try (PDDocument document = new PDDocument()) {
PDAcroForm acroForm = new PDAcroForm(document);
acroForm.setDefaultResources(new PDResources());
document.getDocumentCatalog().setAcroForm(acroForm);
for (int i = 0; i < pageCount; i++) {
PDPage page = new PDPage(PDRectangle.A4);
document.addPage(page);
PDTextField field = new PDTextField(acroForm);
field.setPartialName("field_" + i);
PDAnnotationWidget widget = new PDAnnotationWidget();
widget.setRectangle(new PDRectangle(50, 50, 100, 20));
widget.setPage(page);
field.setWidgets(List.of(widget));
acroForm.getFields().add(field);
page.getAnnotations().add(widget);
}
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
return out.toByteArray();
}
}
}
private static byte[] buildPdfWithMultiWidgetField() throws IOException {
try (PDDocument document = new PDDocument()) {
PDAcroForm acroForm = new PDAcroForm(document);
acroForm.setDefaultResources(new PDResources());
document.getDocumentCatalog().setAcroForm(acroForm);
List<PDAnnotationWidget> widgets = new ArrayList<>();
for (int i = 0; i < 3; i++) {
PDPage page = new PDPage(PDRectangle.A4);
document.addPage(page);
PDAnnotationWidget widget = new PDAnnotationWidget();
widget.setRectangle(new PDRectangle(50, 50, 100, 20));
widget.setPage(page);
page.getAnnotations().add(widget);
widgets.add(widget);
}
PDTextField field = new PDTextField(acroForm);
field.setPartialName("multi");
field.setWidgets(widgets);
acroForm.getFields().add(field);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
return out.toByteArray();
}
}
}
private static byte[] buildPdfWithNestedFields() throws IOException {
try (PDDocument document = new PDDocument()) {
PDAcroForm acroForm = new PDAcroForm(document);
acroForm.setDefaultResources(new PDResources());
document.getDocumentCatalog().setAcroForm(acroForm);
PDPage pageA = new PDPage(PDRectangle.A4);
PDPage pageB = new PDPage(PDRectangle.A4);
document.addPage(pageA);
document.addPage(pageB);
PDNonTerminalField group = new PDNonTerminalField(acroForm);
group.setPartialName("group");
PDTextField kept = new PDTextField(acroForm);
kept.setPartialName("kept");
PDAnnotationWidget keptWidget = new PDAnnotationWidget();
keptWidget.setRectangle(new PDRectangle(50, 50, 100, 20));
keptWidget.setPage(pageA);
kept.setWidgets(List.of(keptWidget));
pageA.getAnnotations().add(keptWidget);
PDTextField dropped = new PDTextField(acroForm);
dropped.setPartialName("dropped");
PDAnnotationWidget droppedWidget = new PDAnnotationWidget();
droppedWidget.setRectangle(new PDRectangle(50, 100, 100, 20));
droppedWidget.setPage(pageB);
dropped.setWidgets(List.of(droppedWidget));
pageB.getAnnotations().add(droppedWidget);
group.setChildren(List.of(kept, dropped));
acroForm.getFields().add(group);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
document.save(out);
return out.toByteArray();
}
}
}
}
+6
View File
@@ -87,6 +87,12 @@ dependencies {
implementation 'com.sun.xml.bind:jaxb-core:4.0.7'
implementation 'org.apache.poi:poi-ooxml:5.5.1'
// https://mvnrepository.com/artifact/technology.tabula/tabula
implementation ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
// CVE-2022-25647: Explicit gson 2.13.2 to prevent unsafe deserialization (tabula would pull 2.8.7)
implementation 'com.google.code.gson:gson:2.13.2'
implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4'
@@ -6,11 +6,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -30,7 +27,6 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.FormUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
@@ -71,7 +67,6 @@ public class RearrangePagesPDFController {
int pageIndex = pagesToRemove.get(i);
document.removePage(pageIndex);
}
FormUtils.pruneOrphanedFormFields(document);
return WebResponseUtils.pdfDocToWebResponse(
document,
GeneralUtils.generateFilename(
@@ -270,17 +265,6 @@ public class RearrangePagesPDFController {
rearrangedDocument.addPage(page);
}
PDDocumentCatalog sourceCatalog = document.getDocumentCatalog();
if (sourceCatalog != null) {
PDAcroForm sourceForm = sourceCatalog.getAcroForm(null);
if (sourceForm != null) {
rearrangedDocument
.getDocumentCatalog()
.getCOSObject()
.setItem(COSName.ACRO_FORM, sourceForm.getCOSObject());
}
}
return WebResponseUtils.pdfDocToWebResponse(
rearrangedDocument,
GeneralUtils.generateFilename(
@@ -1,13 +1,9 @@
package stirling.software.SPDF.controller.api;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -30,7 +26,6 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.FormUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@@ -59,21 +54,9 @@ public class SplitPDFController {
MultipartFile file = request.getFileInput();
TempFile outputTempFile = new TempFile(tempFileManager, ".zip");
try {
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf")) {
Files.copy(
file.getInputStream(),
sourceTempFile.getPath(),
StandardCopyOption.REPLACE_EXISTING);
int totalPages;
List<Integer> pageNumbers;
boolean hasForm;
try (PDDocument document =
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
totalPages = document.getNumberOfPages();
pageNumbers = request.getPageNumbersList(document, false);
hasForm = document.getDocumentCatalog().getAcroForm(null) != null;
}
try (PDDocument document = pdfDocumentFactory.load(file)) {
int totalPages = document.getNumberOfPages();
List<Integer> pageNumbers = request.getPageNumbersList(document, false);
if (!pageNumbers.contains(totalPages - 1)) {
pageNumbers = new ArrayList<>(pageNumbers);
pageNumbers.add(totalPages - 1);
@@ -86,16 +69,33 @@ public class SplitPDFController {
String baseFilename = GeneralUtils.removeExtension(file.getOriginalFilename());
try (ZipOutputStream zipOut =
new ZipOutputStream(Files.newOutputStream(outputTempFile.getPath()))) {
if (hasForm) {
writeSplitsViaReload(
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
} else {
writeSplitsViaSharedSource(
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
int previousPageNumber = 0;
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
int splitPoint = pageNumbers.get(splitIndex);
try (PDDocument splitDocument =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(document)) {
for (int i = previousPageNumber; i <= splitPoint; i++) {
splitDocument.addPage(document.getPage(i));
log.debug("Adding page {} to split document", i);
}
previousPageNumber = splitPoint + 1;
String fileName = baseFilename + "_" + (splitIndex + 1) + ".pdf";
zipOut.putNextEntry(new ZipEntry(fileName));
splitDocument.save(zipOut);
zipOut.closeEntry();
log.debug("Wrote split document {} to zip file", fileName);
} catch (Exception e) {
ExceptionUtils.logException("document splitting and saving", e);
throw e;
}
}
}
}
log.debug(
"Successfully created zip file with split documents: {}",
outputTempFile.getPath().toString());
String zipFilename =
GeneralUtils.generateFilename(file.getOriginalFilename(), "_split.zip");
return WebResponseUtils.zipFileToWebResponse(outputTempFile, zipFilename);
@@ -104,60 +104,4 @@ public class SplitPDFController {
throw e;
}
}
private void writeSplitsViaReload(
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
throws IOException {
int previousPageNumber = 0;
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
int splitPoint = pageNumbers.get(splitIndex);
Set<Integer> keep = new HashSet<>();
for (int i = previousPageNumber; i <= splitPoint; i++) {
keep.add(i);
}
previousPageNumber = splitPoint + 1;
try (PDDocument splitDoc = pdfDocumentFactory.load(source)) {
for (int p = splitDoc.getNumberOfPages() - 1; p >= 0; p--) {
if (!keep.contains(p)) {
splitDoc.removePage(p);
}
}
FormUtils.pruneOrphanedFormFields(splitDoc);
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
} catch (Exception e) {
ExceptionUtils.logException("document splitting and saving", e);
throw e;
}
}
}
private void writeSplitsViaSharedSource(
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
throws IOException {
try (PDDocument sourceDoc = pdfDocumentFactory.load(source)) {
int previousPageNumber = 0;
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
int splitPoint = pageNumbers.get(splitIndex);
try (PDDocument splitDoc =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDoc)) {
for (int i = previousPageNumber; i <= splitPoint; i++) {
splitDoc.addPage(sourceDoc.getPage(i));
}
previousPageNumber = splitPoint + 1;
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
} catch (Exception e) {
ExceptionUtils.logException("document splitting and saving", e);
throw e;
}
}
}
}
private void writeEntry(ZipOutputStream zipOut, String baseFilename, int index, PDDocument doc)
throws IOException {
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + index + ".pdf"));
doc.save(zipOut);
zipOut.closeEntry();
}
}
@@ -1,14 +1,8 @@
package stirling.software.SPDF.controller.api;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -22,6 +16,7 @@ import org.springframework.web.multipart.MultipartFile;
import io.swagger.v3.oas.annotations.Operation;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -31,7 +26,6 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.FormUtils;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
@@ -60,40 +54,52 @@ public class SplitPdfBySizeController {
public ResponseEntity<Resource> autoSplitPdf(
@ModelAttribute SplitPdfBySizeOrCountRequest request) throws Exception {
log.debug("Starting PDF split process with request: {}", request);
MultipartFile file = request.getFileInput();
String filename = GeneralUtils.generateFilename(file.getOriginalFilename(), "");
log.debug("Base filename for output: {}", filename);
TempFile zipTempFile = new TempFile(tempFileManager, ".zip");
try {
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf");
ZipOutputStream zipOut =
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()))) {
Files.copy(
file.getInputStream(),
sourceTempFile.getPath(),
StandardCopyOption.REPLACE_EXISTING);
log.debug("Created temporary managed zip file: {}", zipTempFile.getPath());
log.debug("Creating ZIP output stream");
try (ZipOutputStream zipOut =
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()));
PDDocument sourceDocument = pdfDocumentFactory.load(file)) {
log.debug(
"Successfully loaded PDF with {} pages", sourceDocument.getNumberOfPages());
try (PDDocument sourceDocument =
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
boolean hasForm = sourceDocument.getDocumentCatalog().getAcroForm(null) != null;
List<List<Integer>> ranges = computeRanges(request, sourceDocument);
int type = request.getSplitType();
String value = request.getSplitValue();
log.debug("Split type: {}, Split value: {}", type, value);
int fileIndex = 1;
for (List<Integer> range : ranges) {
if (range.isEmpty()) {
continue;
}
if (hasForm) {
writeRangeViaReload(
sourceTempFile.getFile(), range, zipOut, filename, fileIndex++);
} else {
writeRangeViaSharedSource(
sourceDocument, range, zipOut, filename, fileIndex++);
}
}
if (type == 0) {
log.debug("Processing split by size");
long maxBytes = GeneralUtils.convertSizeToBytes(value);
log.debug("Max bytes per document: {}", maxBytes);
handleSplitBySize(sourceDocument, maxBytes, zipOut, filename);
} else if (type == 1) {
log.debug("Processing split by page count");
int pageCount = Integer.parseInt(value);
log.debug("Pages per document: {}", pageCount);
handleSplitByPageCount(sourceDocument, pageCount, zipOut, filename);
} else if (type == 2) {
log.debug("Processing split by document count");
int documentCount = Integer.parseInt(value);
log.debug("Total number of documents: {}", documentCount);
handleSplitByDocCount(sourceDocument, documentCount, zipOut, filename);
} else {
log.error("Invalid split type: {}", type);
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument",
"Invalid argument: {0}",
"split type: " + type);
}
log.debug("PDF splitting completed successfully");
}
log.debug("Returning streaming response for zip file");
return WebResponseUtils.zipFileToWebResponse(zipTempFile, filename + ".zip");
} catch (Exception e) {
ExceptionUtils.logException("PDF splitting process", e);
@@ -102,198 +108,387 @@ public class SplitPdfBySizeController {
}
}
private List<List<Integer>> computeRanges(
SplitPdfBySizeOrCountRequest request, PDDocument sourceDocument) throws IOException {
int type = request.getSplitType();
String value = request.getSplitValue();
if (type == 0) {
return computeSizeRanges(sourceDocument, GeneralUtils.convertSizeToBytes(value));
} else if (type == 1) {
return computePageCountRanges(sourceDocument, Integer.parseInt(value));
} else if (type == 2) {
return computeDocCountRanges(sourceDocument, Integer.parseInt(value));
}
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument", "Invalid argument: {0}", "split type: " + type);
}
private void writeRangeViaReload(
File sourceFile,
List<Integer> keepIndices,
ZipOutputStream zipOut,
String baseFilename,
int fileIndex)
private void handleSplitBySize(
PDDocument sourceDocument, long maxBytes, ZipOutputStream zipOut, String baseFilename)
throws IOException {
Set<Integer> keep = new HashSet<>(keepIndices);
try (PDDocument doc = pdfDocumentFactory.load(sourceFile)) {
for (int i = doc.getNumberOfPages() - 1; i >= 0; i--) {
if (!keep.contains(i)) {
doc.removePage(i);
log.debug("Starting handleSplitBySize with maxBytes={}", maxBytes);
@Getter
class DocHolder implements AutoCloseable {
private PDDocument doc;
public DocHolder(PDDocument doc) {
this.doc = doc;
}
public void setDoc(PDDocument doc) {
if (this.doc != null) {
try {
this.doc.close();
} catch (IOException e) {
log.error("Error closing document", e);
}
}
this.doc = doc;
}
@Override
public void close() throws IOException {
if (doc != null) {
doc.close();
}
}
FormUtils.pruneOrphanedFormFields(doc);
writeEntry(zipOut, baseFilename, fileIndex, doc);
}
}
private void writeRangeViaSharedSource(
PDDocument sourceDocument,
List<Integer> keepIndices,
ZipOutputStream zipOut,
String baseFilename,
int fileIndex)
throws IOException {
try (PDDocument doc =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument)) {
for (int p : keepIndices) {
doc.addPage(sourceDocument.getPage(p));
}
writeEntry(zipOut, baseFilename, fileIndex, doc);
}
}
int fileIndex = 1;
try (DocHolder holder =
new DocHolder(
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument))) {
int totalPages = sourceDocument.getNumberOfPages();
int pageAdded = 0;
private void writeEntry(
ZipOutputStream zipOut, String baseFilename, int fileIndex, PDDocument doc)
throws IOException {
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + fileIndex + ".pdf"));
doc.save(zipOut);
zipOut.closeEntry();
}
// Smart size check frequency - check more often with larger documents
int baseCheckFrequency = 5;
/** Page-index ranges each output should contain. AcroForm overhead isn't modeled. */
private List<List<Integer>> computeSizeRanges(PDDocument sourceDocument, long maxBytes)
throws IOException {
List<List<Integer>> ranges = new ArrayList<>();
List<Integer> currentRange = new ArrayList<>();
int totalPages = sourceDocument.getNumberOfPages();
int baseCheckFrequency = 5;
PDDocument scratch = new PDDocument();
try {
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
PDPage page = sourceDocument.getPage(pageIndex);
scratch.addPage(new PDPage(page.getCOSObject()));
currentRange.add(pageIndex);
log.debug("Processing page {} of {}", pageIndex + 1, totalPages);
int pageAdded = currentRange.size();
// Add the page to current document
PDPage newPage = new PDPage(page.getCOSObject());
holder.getDoc().addPage(newPage);
pageAdded++;
// Dynamic size checking based on document size and page count
boolean shouldCheckSize =
(pageAdded % baseCheckFrequency == 0)
|| (pageIndex == totalPages - 1)
|| (pageAdded >= 20);
if (!shouldCheckSize) {
continue;
}
|| (pageAdded >= 20); // Always check after 20 pages
long actualSize;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
scratch.save(out);
actualSize = out.size();
}
if (shouldCheckSize) {
log.debug("Performing size check after {} pages", pageAdded);
long actualSize;
try (ByteArrayOutputStream checkSizeStream = new ByteArrayOutputStream()) {
holder.getDoc().save(checkSizeStream);
actualSize = checkSizeStream.size();
}
log.debug(
"Current document size: {} bytes (max: {} bytes)",
actualSize,
maxBytes);
if (actualSize > maxBytes) {
if (scratch.getNumberOfPages() > 1) {
scratch.removePage(scratch.getNumberOfPages() - 1);
currentRange.remove(currentRange.size() - 1);
pageIndex--; // retry this page in the next chunk
if (actualSize > maxBytes) {
// We exceeded the limit - remove the last page and save
if (holder.getDoc().getNumberOfPages() > 1) {
holder.getDoc().removePage(holder.getDoc().getNumberOfPages() - 1);
pageIndex--; // Process this page again in the next document
log.debug("Size limit exceeded - removed last page");
}
log.debug(
"Saving document with {} pages as part {}",
holder.getDoc().getNumberOfPages(),
fileIndex);
saveDocumentToZip(holder.getDoc(), zipOut, baseFilename, fileIndex++);
holder.setDoc(new PDDocument());
pageAdded = 0;
} else if (pageIndex < totalPages - 1) {
// We're under the limit, calculate if we might fit more pages
// Try to predict how many more similar pages might fit
if (actualSize < maxBytes * 0.75 && pageAdded > 0) {
// Rather than using a ratio, look ahead to test actual upcoming pages
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
if (pagesToLookAhead > 0) {
log.debug(
"Testing {} upcoming pages for potential addition",
pagesToLookAhead);
// Create a temp document with current pages + look-ahead pages
try (PDDocument testDoc = new PDDocument()) {
// First copy existing pages
for (int i = 0; i < holder.getDoc().getNumberOfPages(); i++) {
testDoc.addPage(
new PDPage(
holder.getDoc().getPage(i).getCOSObject()));
}
// Try adding look-ahead pages one by one
int extraPagesAdded = 0;
for (int i = 0; i < pagesToLookAhead; i++) {
int testPageIndex = pageIndex + 1 + i;
PDPage testPage = sourceDocument.getPage(testPageIndex);
testDoc.addPage(new PDPage(testPage.getCOSObject()));
// Check if we're still under size
long testSize;
try (ByteArrayOutputStream testStream =
new ByteArrayOutputStream()) {
testDoc.save(testStream);
testSize = testStream.size();
}
if (testSize <= maxBytes) {
extraPagesAdded++;
log.debug(
"Test: Can add page {} (size would be {})",
testPageIndex + 1,
testSize);
} else {
log.debug(
"Test: Cannot add page {} (size would be {})",
testPageIndex + 1,
testSize);
break;
}
}
// Add the pages we verified would fit
if (extraPagesAdded > 0) {
log.debug(
"Adding {} verified pages ahead", extraPagesAdded);
for (int i = 0; i < extraPagesAdded; i++) {
int extraPageIndex = pageIndex + 1 + i;
PDPage extraPage =
sourceDocument.getPage(extraPageIndex);
holder.getDoc()
.addPage(new PDPage(extraPage.getCOSObject()));
}
pageIndex += extraPagesAdded;
pageAdded += extraPagesAdded;
}
}
}
}
}
ranges.add(new ArrayList<>(currentRange));
currentRange.clear();
scratch.close();
scratch = new PDDocument();
} else if (pageIndex < totalPages - 1 && actualSize < maxBytes * 0.75) {
int extraPagesAdded =
lookAheadFit(scratch, sourceDocument, pageIndex, maxBytes);
for (int i = 0; i < extraPagesAdded; i++) {
int extra = pageIndex + 1 + i;
scratch.addPage(new PDPage(sourceDocument.getPage(extra).getCOSObject()));
currentRange.add(extra);
}
pageIndex += extraPagesAdded;
}
}
if (!currentRange.isEmpty()) {
ranges.add(new ArrayList<>(currentRange));
// Save final document if it has any pages
if (holder.getDoc() != null && holder.getDoc().getNumberOfPages() > 0) {
log.debug(
"Saving final document with {} pages as part {}",
holder.getDoc().getNumberOfPages(),
fileIndex);
saveDocumentToZip(holder.getDoc(), zipOut, baseFilename, fileIndex++);
holder.setDoc(null);
}
}
log.debug("Completed handleSplitBySize with {} document parts created", fileIndex - 1);
}
private void handleSplitByPageCount(
PDDocument sourceDocument, int pageCount, ZipOutputStream zipOut, String baseFilename)
throws IOException {
log.debug("Starting handleSplitByPageCount with pageCount={}", pageCount);
int currentPageCount = 0;
PDDocument currentDoc = null;
int fileIndex = 1;
try {
log.debug("Creating initial output document");
try {
currentDoc = pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument);
log.debug("Successfully created initial output document");
} catch (Exception e) {
ExceptionUtils.logException("initial output document creation", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
int pageIndex = 0;
int totalPages = sourceDocument.getNumberOfPages();
log.debug("Processing {} pages", totalPages);
try {
for (PDPage page : sourceDocument.getPages()) {
pageIndex++;
log.debug("Processing page {} of {}", pageIndex, totalPages);
try {
log.debug("Adding page {} to current document", pageIndex);
currentDoc.addPage(page);
log.debug("Successfully added page {} to current document", pageIndex);
} catch (Exception e) {
log.error("Error adding page {} to current document", pageIndex, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
currentPageCount++;
log.debug("Current page count: {}/{}", currentPageCount, pageCount);
if (currentPageCount == pageCount) {
log.debug(
"Reached target page count ({}), saving current document as part {}",
pageCount,
fileIndex);
try {
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
currentDoc = null; // Document is closed by saveDocumentToZip
log.debug("Successfully saved document part {}", fileIndex - 1);
} catch (Exception e) {
log.error("Error saving document part {}", fileIndex - 1, e);
throw e;
}
try {
log.debug("Creating new document for next part");
currentDoc = new PDDocument();
log.debug("Successfully created new document");
} catch (Exception e) {
log.error("Error creating new document for next part", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
currentPageCount = 0;
log.debug("Reset current page count to 0");
}
}
} catch (Exception e) {
log.error("Error iterating through pages", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
// Add the last document if it contains any pages
try {
if (currentDoc != null && currentDoc.getPages().getCount() != 0) {
log.debug(
"Saving final document with {} pages as part {}",
currentDoc.getPages().getCount(),
fileIndex);
try {
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
currentDoc = null; // Document is closed by saveDocumentToZip
log.debug("Successfully saved final document part {}", fileIndex - 1);
} catch (Exception e) {
log.error("Error saving final document part {}", fileIndex - 1, e);
throw e;
}
} else {
log.debug("Final document has no pages, skipping");
}
} catch (Exception e) {
log.error("Error checking or saving final document", e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
} finally {
scratch.close();
if (currentDoc != null) {
try {
log.debug("Closing remaining document");
currentDoc.close();
log.debug("Successfully closed remaining document");
} catch (Exception e) {
log.error("Error closing remaining document", e);
}
}
}
return ranges;
log.debug("Completed handleSplitByPageCount with {} document parts created", fileIndex - 1);
}
/** Speculatively tries up to 5 next pages; returns how many fit under {@code maxBytes}. */
private int lookAheadFit(PDDocument scratch, PDDocument source, int pageIndex, long maxBytes)
private void handleSplitByDocCount(
PDDocument sourceDocument,
int documentCount,
ZipOutputStream zipOut,
String baseFilename)
throws IOException {
int totalPages = source.getNumberOfPages();
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
if (pagesToLookAhead == 0) {
return 0;
}
log.debug("Starting handleSplitByDocCount with documentCount={}", documentCount);
int totalPageCount = sourceDocument.getNumberOfPages();
log.debug("Total pages in source document: {}", totalPageCount);
int extraPagesAdded = 0;
try (PDDocument testDoc = new PDDocument()) {
for (int i = 0; i < scratch.getNumberOfPages(); i++) {
testDoc.addPage(new PDPage(scratch.getPage(i).getCOSObject()));
}
for (int i = 0; i < pagesToLookAhead; i++) {
testDoc.addPage(new PDPage(source.getPage(pageIndex + 1 + i).getCOSObject()));
long testSize;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
testDoc.save(out);
testSize = out.size();
}
if (testSize > maxBytes) {
break;
}
extraPagesAdded++;
}
}
return extraPagesAdded;
}
int pagesPerDocument = totalPageCount / documentCount;
int extraPages = totalPageCount % documentCount;
log.debug("Pages per document: {}, Extra pages: {}", pagesPerDocument, extraPages);
private List<List<Integer>> computePageCountRanges(PDDocument sourceDocument, int pageCount) {
if (pageCount <= 0) {
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument", "Invalid argument: {0}", "page count: " + pageCount);
}
int totalPages = sourceDocument.getNumberOfPages();
List<List<Integer>> ranges = new ArrayList<>();
List<Integer> current = new ArrayList<>(pageCount);
for (int i = 0; i < totalPages; i++) {
current.add(i);
if (current.size() == pageCount) {
ranges.add(current);
current = new ArrayList<>(pageCount);
}
}
if (!current.isEmpty()) {
ranges.add(current);
}
return ranges;
}
int currentPageIndex = 0;
int fileIndex = 1;
private List<List<Integer>> computeDocCountRanges(
PDDocument sourceDocument, int documentCount) {
if (documentCount <= 0) {
throw ExceptionUtils.createIllegalArgumentException(
"error.invalidArgument",
"Invalid argument: {0}",
"document count: " + documentCount);
}
int totalPages = sourceDocument.getNumberOfPages();
int pagesPerDocument = totalPages / documentCount;
int extraPages = totalPages % documentCount;
List<List<Integer>> ranges = new ArrayList<>();
int cursor = 0;
for (int i = 0; i < documentCount; i++) {
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
List<Integer> range = new ArrayList<>(pagesToAdd);
for (int j = 0; j < pagesToAdd; j++) {
range.add(cursor++);
log.debug("Creating document {} of {}", i + 1, documentCount);
PDDocument currentDoc = null;
try {
currentDoc = pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument);
log.debug("Successfully created document {} of {}", i + 1, documentCount);
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
log.debug("Adding {} pages to document {}", pagesToAdd, i + 1);
for (int j = 0; j < pagesToAdd; j++) {
try {
log.debug(
"Adding page {} (index {}) to document {}",
j + 1,
currentPageIndex,
i + 1);
currentDoc.addPage(sourceDocument.getPage(currentPageIndex));
log.debug("Successfully added page {} to document {}", j + 1, i + 1);
currentPageIndex++;
} catch (Exception e) {
log.error("Error adding page {} to document {}", j + 1, i + 1, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
}
try {
log.debug("Saving document {} with {} pages", i + 1, pagesToAdd);
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
// saveDocumentToZip closes the document
currentDoc = null;
log.debug("Successfully saved document {}", i + 1);
} catch (Exception e) {
log.error("Error saving document {}", i + 1, e);
throw e;
}
} catch (Exception e) {
log.error("Error creating document {} of {}", i + 1, documentCount, e);
throw ExceptionUtils.createFileProcessingException("split", e);
} finally {
if (currentDoc != null) {
try {
currentDoc.close();
} catch (IOException e) {
log.error("Error closing document {} of {}", i + 1, documentCount, e);
}
}
}
}
log.debug("Completed handleSplitByDocCount with {} documents created", documentCount);
}
private void saveDocumentToZip(
PDDocument document, ZipOutputStream zipOut, String baseFilename, int index)
throws IOException {
log.debug("Starting saveDocumentToZip for document part {}", index);
try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
try (PDDocument doc = document) {
log.debug("Saving document part {} to byte array", index);
doc.save(outStream);
log.debug(
"Successfully saved document part {} ({} bytes)", index, outStream.size());
} catch (Exception e) {
log.error("Error saving document part {} to byte array", index, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
try {
// Create a new zip entry
String entryName = baseFilename + "_" + index + ".pdf";
log.debug("Creating ZIP entry: {}", entryName);
ZipEntry zipEntry = new ZipEntry(entryName);
zipOut.putNextEntry(zipEntry);
byte[] bytes = outStream.toByteArray();
log.debug("Writing {} bytes to ZIP entry", bytes.length);
zipOut.write(bytes);
log.debug("Closing ZIP entry");
zipOut.closeEntry();
log.debug("Successfully added document part {} to ZIP", index);
} catch (Exception e) {
log.error("Error adding document part {} to ZIP", index, e);
throw ExceptionUtils.createFileProcessingException("split", e);
}
ranges.add(range);
}
return ranges;
}
}
@@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.QuoteMode;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.http.ContentDisposition;
@@ -26,21 +26,24 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.config.swagger.CsvConversionResponse;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.SPDF.pdf.FlexibleCSVWriter;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.ConvertApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.WebResponseUtils;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.Table;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
@ConvertApi
@Slf4j
@RequiredArgsConstructor
public class ExtractCSVController {
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final TabulaTableParser tabulaTableParser;
@AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@CsvConversionResponse
@@ -55,23 +58,24 @@ public class ExtractCSVController {
try (PDDocument document = pdfDocumentFactory.load(request)) {
List<Integer> pages = request.getPageNumbersList(document, true);
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
CSVFormat format =
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
for (int pageNum : pages) {
log.info("{}", pageNum);
List<TableFragment> fragments = tabulaTableParser.parse(document, pageNum);
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
log.info("{}", pageNum);
Page page = extractor.extract(pageNum);
List<Table> tables = sea.extract(page);
for (int i = 0; i < fragments.size(); i++) {
StringWriter sw = new StringWriter();
try (CSVPrinter printer = format.print(sw)) {
for (List<String> row : fragments.get(i).rawRows()) {
printer.printRecord(row);
}
for (int i = 0; i < tables.size(); i++) {
StringWriter sw = new StringWriter();
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
csvWriter.write(sw, Collections.singletonList(tables.get(i)));
String entryName = generateEntryName(baseName, pageNum, i + 1);
csvEntries.add(new CsvEntry(entryName, sw.toString()));
}
csvEntries.add(
new CsvEntry(
generateEntryName(baseName, pageNum, i + 1), sw.toString()));
}
}
@@ -13,7 +13,8 @@ public class RotatePDFRequest extends PDFFile {
@Schema(
description =
"The clockwise angle by which to rotate all pages in the PDF file. Must be a multiple of 90.",
"The clockwise angle by which to rotate the PDF file. Must be a multiple of"
+ " 90.",
type = "integer",
requiredMode = Schema.RequiredMode.REQUIRED,
allowableValues = {"0", "90", "180", "270"})
@@ -0,0 +1,16 @@
package stirling.software.SPDF.pdf;
import org.apache.commons.csv.CSVFormat;
import technology.tabula.writers.CSVWriter;
public class FlexibleCSVWriter extends CSVWriter {
public FlexibleCSVWriter() {
super();
}
public FlexibleCSVWriter(CSVFormat csvFormat) {
super(csvFormat);
}
}
@@ -3,10 +3,8 @@ package stirling.software.SPDF.controller.api;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -26,6 +24,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.api.SplitPagesRequest;
import stirling.software.common.service.CustomPDFDocumentFactory;
@@ -61,10 +60,8 @@ class SplitPDFControllerTest {
}
private void setupFactory() throws IOException {
when(pdfDocumentFactory.load(any(File.class), eq(true)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(File.class)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(MultipartFile.class)))
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
.thenAnswer(inv -> new PDDocument());
}
@@ -3,10 +3,8 @@ package stirling.software.SPDF.controller.api;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -27,6 +25,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.api.general.SplitPdfBySizeOrCountRequest;
import stirling.software.common.service.CustomPDFDocumentFactory;
@@ -71,8 +70,9 @@ class SplitPdfBySizeControllerTest {
request.setSplitType(1); // Page count
request.setSplitValue("2");
when(pdfDocumentFactory.load(any(File.class), eq(true)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(MultipartFile.class)))
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
.thenAnswer(inv -> new PDDocument());
@@ -104,8 +104,14 @@ class SplitPdfBySizeControllerTest {
request.setSplitType(2); // Document count
request.setSplitValue("3"); // Split into 3 docs (2 pages each)
when(pdfDocumentFactory.load(any(File.class), eq(true)))
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
when(pdfDocumentFactory.load(any(org.springframework.web.multipart.MultipartFile.class)))
.thenAnswer(
inv ->
Loader.loadPDF(
((org.springframework.web.multipart.MultipartFile)
inv.getArgument(0))
.getBytes()));
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
.thenAnswer(inv -> new PDDocument());
@@ -20,7 +20,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.GeneralUtils;
@@ -28,7 +27,6 @@ import stirling.software.common.util.GeneralUtils;
class ExtractCSVControllerTest {
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
@Mock private TabulaTableParser tabulaTableParser;
@InjectMocks private ExtractCSVController controller;
@@ -0,0 +1,25 @@
package stirling.software.SPDF.pdf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.commons.csv.CSVFormat;
import org.junit.jupiter.api.Test;
class FlexibleCSVWriterTest {
@Test
void testDefaultConstructor() {
FlexibleCSVWriter writer = new FlexibleCSVWriter();
assertNotNull(writer, "The FlexibleCSVWriter instance should not be null");
}
@Test
void testConstructorWithCSVFormat() {
CSVFormat csvFormat = CSVFormat.DEFAULT;
FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat);
assertNotNull(
writer,
"The FlexibleCSVWriter instance should not be null when initialized with"
+ " CSVFormat");
}
}
+6
View File
@@ -59,6 +59,12 @@ dependencies {
// https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
// Tabula table extraction — used by MathAuditorOrchestrator
implementation ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
implementation 'com.google.code.gson:gson:2.13.2'
api 'io.micrometer:micrometer-registry-prometheus'
@@ -30,7 +30,6 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.service.TaskManager;
import stirling.software.proprietary.model.api.ai.AiWorkflowProgressEvent;
import stirling.software.proprietary.model.api.ai.AiWorkflowRequest;
import stirling.software.proprietary.model.api.ai.AiWorkflowResponse;
import stirling.software.proprietary.model.api.ai.AiWorkflowResultFile;
@@ -141,32 +140,13 @@ public class AiEngineController {
}
private void runOrchestrationStream(AiWorkflowRequest request, SseEmitter emitter) {
AiWorkflowService.ProgressListener listener =
new AiWorkflowService.ProgressListener() {
@Override
public void onProgress(AiWorkflowProgressEvent event) {
sendEvent(emitter, "progress", event);
}
@Override
public void onHeartbeat() {
// Forward upstream heartbeats so the SSE pipe stays visibly alive between
// real progress events; if the frontend has gone away, sendEvent throws,
// which propagates up through the stream consumer and closes our upstream
// engine connection so the engine can cancel its in-flight workflow.
sendEvent(emitter, "heartbeat", Map.of());
}
};
try {
AiWorkflowResponse result = aiWorkflowService.orchestrate(request, listener);
AiWorkflowResponse result =
aiWorkflowService.orchestrate(
request, progress -> sendEvent(emitter, "progress", progress));
registerFileResultAsJob(result);
sendEvent(emitter, "result", result);
emitter.complete();
} catch (ClientDisconnectedException e) {
// The frontend gave up mid-stream. The exception unwinding through orchestrate()
// already closed the upstream engine connection (engine sees disconnect and cancels).
// The emitter is already toast; nothing useful left to send.
log.debug("Client disconnected mid-stream; aborting workflow", e);
} catch (Exception e) {
log.error("AI orchestration stream failed", e);
// Emit an error frame for the frontend and then complete normally. Using
@@ -212,21 +192,7 @@ public class AiEngineController {
try {
emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON));
} catch (IOException e) {
// Surface the disconnect so the streaming pipeline unwinds: callers higher up close
// the upstream engine connection, which lets the engine cancel its in-flight workflow.
// Without this, the engine would keep producing (and billing for) tokens whose results
// nobody is reading.
throw new ClientDisconnectedException("Client disconnected from SSE stream", e);
}
}
/**
* Thrown by {@link #sendEvent} when the SSE emitter's underlying connection is gone. Treated as
* a signal to abort the workflow, not as an error to report.
*/
private static final class ClientDisconnectedException extends RuntimeException {
ClientDisconnectedException(String message, Throwable cause) {
super(message, cause);
log.debug("Failed to send SSE event (client may have disconnected)", e);
}
}
@@ -1,24 +0,0 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Body for {@code POST /api/v1/documents} on the AI engine. Sent by Java when the engine reports
* {@code need_ingest} and the requested document's extracted content must be stored before the
* workflow can continue.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AiDocumentIngestRequest {
private String documentId;
private String source;
private List<AiPageText> pageText;
}
@@ -1,59 +0,0 @@
package stirling.software.proprietary.model.api.ai;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* Typed engine-emitted progress detail, mirroring the Python {@code ProgressEvent} discriminated
* union (see {@code engine/src/stirling/contracts/progress.py}). Carried inside {@link
* AiWorkflowProgressEvent#getEngineDetail()} for {@link AiWorkflowPhase#ENGINE_PROGRESS} events.
*
* <p>Sealed so adding a new engine-side phase forces a matching subtype on the Java side instead of
* silently passing through as an opaque map. The {@code phase} string is the discriminator and
* stays on the wire so the frontend (which doesn't know about Java's class hierarchy) can switch on
* it.
*/
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.EXISTING_PROPERTY,
property = "phase",
visible = true)
@JsonSubTypes({
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocReadStarted.class,
name = "whole_doc_read_started"),
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocSliceDone.class,
name = "whole_doc_slice_done"),
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocCompressionRound.class,
name = "whole_doc_compression_round"),
@JsonSubTypes.Type(
value = AiEngineProgressDetail.WholeDocReadDone.class,
name = "whole_doc_read_done"),
})
@JsonIgnoreProperties(ignoreUnknown = true)
public sealed interface AiEngineProgressDetail {
String phase();
record WholeDocReadStarted(String phase, String question, int pages, int slices)
implements AiEngineProgressDetail {}
record WholeDocSliceDone(
String phase,
int completed,
int total,
String pages,
int durationMs,
int excerpts,
int facts)
implements AiEngineProgressDetail {}
record WholeDocCompressionRound(String phase, int roundNumber, int notesIn, int groups)
implements AiEngineProgressDetail {}
record WholeDocReadDone(String phase, int completed, int slices, double durationSeconds)
implements AiEngineProgressDetail {}
}
@@ -0,0 +1,24 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Body for {@code POST /api/v1/rag/documents} on the AI engine. Sent by Java when the engine
* reports {@code need_ingest} and the requested document's extracted content must be stored before
* the workflow can continue.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AiRagIngestRequest {
private String documentId;
private String source;
private List<AiRagPageText> pageText;
}
@@ -4,11 +4,11 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/** A single page of extracted text for document ingest requests. */
/** A single page of extracted text for RAG ingest requests. */
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AiPageText {
public class AiRagPageText {
private int pageNumber;
@@ -20,8 +20,7 @@ public enum AiWorkflowOutcome {
TOOL_CALL("tool_call"),
COMPLETED("completed"),
UNSUPPORTED_CAPABILITY("unsupported_capability"),
CANNOT_CONTINUE("cannot_continue"),
GENERATE_FILE("generate_file");
CANNOT_CONTINUE("cannot_continue");
private final String value;
@@ -9,13 +9,7 @@ public enum AiWorkflowPhase {
CALLING_ENGINE("calling_engine"),
EXTRACTING_CONTENT("extracting_content"),
EXECUTING_TOOL("executing_tool"),
PROCESSING("processing"),
/**
* Generic engine-emitted progress event (e.g. chunked-reasoner slice progress). The original
* engine event JSON is carried in {@link AiWorkflowProgressEvent#getEngineDetail()}, including
* a specific {@code phase} string the frontend can switch on.
*/
ENGINE_PROGRESS("engine_progress");
PROCESSING("processing");
private final String value;
@@ -23,17 +23,8 @@ public class AiWorkflowProgressEvent {
/** Total number of plan steps, for {@link AiWorkflowPhase#EXECUTING_TOOL} events. */
private Integer stepCount;
/**
* Engine-emitted event payload, for {@link AiWorkflowPhase#ENGINE_PROGRESS} events. The payload
* is a typed subtype keyed on its {@code phase} string (e.g. {@code "whole_doc_slice_done"})
* carrying phase-specific fields (slice index, page range, durations, etc.) that the frontend
* can render as detailed progress.
*/
private AiEngineProgressDetail engineDetail;
public static AiWorkflowProgressEvent of(AiWorkflowPhase phase) {
return new AiWorkflowProgressEvent(
phase, System.currentTimeMillis(), null, null, null, null);
return new AiWorkflowProgressEvent(phase, System.currentTimeMillis(), null, null, null);
}
public static AiWorkflowProgressEvent executingTool(String tool, int stepIndex, int stepCount) {
@@ -42,17 +33,6 @@ public class AiWorkflowProgressEvent {
System.currentTimeMillis(),
tool,
stepIndex,
stepCount,
null);
}
public static AiWorkflowProgressEvent engineProgress(AiEngineProgressDetail detail) {
return new AiWorkflowProgressEvent(
AiWorkflowPhase.ENGINE_PROGRESS,
System.currentTimeMillis(),
null,
null,
null,
detail);
stepCount);
}
}
@@ -4,8 +4,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@@ -22,14 +20,6 @@ public class AiWorkflowResponse {
@Schema(description = "Answer returned by the AI workflow when applicable")
private String answer;
@JsonProperty("content")
@Schema(description = "Text content to package as a file (generate_file outcomes)")
private String generatedContent;
@JsonProperty("filename")
@Schema(description = "Desired output filename for generate_file outcomes")
private String generatedFilename;
@Schema(description = "Summary returned by the AI workflow when applicable")
private String summary;
@@ -0,0 +1,17 @@
package stirling.software.proprietary.pdf;
import org.apache.commons.csv.CSVFormat;
import technology.tabula.writers.CSVWriter;
/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */
public class FlexibleCSVWriter extends CSVWriter {
public FlexibleCSVWriter() {
super();
}
public FlexibleCSVWriter(CSVFormat csvFormat) {
super(csvFormat);
}
}
@@ -7,8 +7,6 @@ import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpTimeoutException;
import java.time.Duration;
import java.util.function.Consumer;
import java.util.stream.Stream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@@ -85,70 +83,6 @@ public class AiEngineClient {
return response.body();
}
/**
* POST a JSON body and consume the response as a stream of NDJSON lines. Each line is passed to
* {@code lineConsumer} in arrival order; the call returns when the engine closes the stream.
*
* <p>This is the right shape for long-running orchestrator calls that emit incremental
* progress. The total HTTP timeout is the long-running timeout (typically 600s+), but in
* practice line arrival keeps the connection logically alive: as long as the engine emits
* events, the work is progressing. Genuine engine hangs still hit the total timeout.
*/
public void streamPost(String path, String jsonBody, Consumer<String> lineConsumer)
throws IOException {
ApplicationProperties.AiEngine config = applicationProperties.getAiEngine();
if (!config.isEnabled()) {
throw new ResponseStatusException(
HttpStatus.SERVICE_UNAVAILABLE, "AI engine is not enabled");
}
String url = config.getUrl().stripTrailing() + path;
Duration timeout = Duration.ofSeconds(config.getLongRunningTimeoutSeconds());
log.debug(
"Proxying AI engine streaming request to {} (timeout {}s)",
url,
timeout.toSeconds());
HttpRequest request =
HttpRequest.newBuilder()
.uri(URI.create(url))
.header("Content-Type", "application/json")
.header("Accept", "application/x-ndjson")
.timeout(timeout)
.POST(HttpRequest.BodyPublishers.ofString(jsonBody))
.build();
HttpResponse<Stream<String>> response;
try {
response = httpClient.send(request, HttpResponse.BodyHandlers.ofLines());
} catch (HttpTimeoutException e) {
throw new ResponseStatusException(HttpStatus.GATEWAY_TIMEOUT, "AI engine timed out", e);
} catch (IOException e) {
throw new ResponseStatusException(
HttpStatus.SERVICE_UNAVAILABLE, "AI engine unreachable: " + e.getMessage(), e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ResponseStatusException(
HttpStatus.SERVICE_UNAVAILABLE, "AI engine request was interrupted");
}
int status = response.statusCode();
if (status >= 400) {
throw new ResponseStatusException(
HttpStatus.valueOf(status >= 500 ? 502 : status),
"AI engine returned error: " + status);
}
try (Stream<String> lines = response.body()) {
lines.forEach(
line -> {
if (!line.isEmpty()) {
lineConsumer.accept(line);
}
});
}
}
public String get(String path) throws IOException {
ApplicationProperties.AiEngine config = applicationProperties.getAiEngine();
if (!config.isEnabled()) {
@@ -37,10 +37,9 @@ import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.ZipExtractionUtils;
import stirling.software.proprietary.model.api.ai.AiConversationMessage;
import stirling.software.proprietary.model.api.ai.AiDocumentIngestRequest;
import stirling.software.proprietary.model.api.ai.AiEngineProgressDetail;
import stirling.software.proprietary.model.api.ai.AiFile;
import stirling.software.proprietary.model.api.ai.AiPageText;
import stirling.software.proprietary.model.api.ai.AiRagIngestRequest;
import stirling.software.proprietary.model.api.ai.AiRagPageText;
import stirling.software.proprietary.model.api.ai.AiWorkflowFileInput;
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
import stirling.software.proprietary.model.api.ai.AiWorkflowOutcome;
@@ -62,7 +61,7 @@ import tools.jackson.databind.ObjectMapper;
@RequiredArgsConstructor
public class AiWorkflowService {
private static final String DOCUMENTS_ENDPOINT = "/api/v1/documents";
private static final String RAG_DOCUMENTS_ENDPOINT = "/api/v1/rag/documents";
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final AiEngineClient aiEngineClient;
@@ -78,14 +77,6 @@ public class AiWorkflowService {
@FunctionalInterface
public interface ProgressListener {
void onProgress(AiWorkflowProgressEvent event);
/**
* Called when the engine emits a keep-alive heartbeat. Default is a no-op; consumers that
* forward to a downstream connection (e.g. an SSE emitter) override this to push a
* heartbeat through, so the next downstream-disconnect surfaces immediately rather than
* waiting for the next real progress event.
*/
default void onHeartbeat() {}
}
private static final ProgressListener NOOP_LISTENER = event -> {};
@@ -153,14 +144,13 @@ public class AiWorkflowService {
ProgressListener listener)
throws IOException {
listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.CALLING_ENGINE));
AiWorkflowResponse response = invokeOrchestrator(request, listener);
AiWorkflowResponse response = invokeOrchestrator(request);
return switch (response.getOutcome()) {
case NEED_CONTENT -> onNeedContent(response, filesById, request, listener);
case NEED_INGEST -> onNeedIngest(response, filesById, request, listener);
case TOOL_CALL -> onToolCall(response, filesById, listener);
case PLAN -> onPlan(response, filesById, request, listener);
case ANSWER -> onAnswer(response, filesById, request, listener);
case GENERATE_FILE -> onGenerateFile(response, listener);
case NOT_FOUND,
NEED_CLARIFICATION,
CANNOT_DO,
@@ -288,22 +278,22 @@ public class AiWorkflowService {
}
private void ingestFile(AiFile file, MultipartFile multipartFile) throws IOException {
List<AiPageText> pages = new ArrayList<>();
List<AiRagPageText> pages = new ArrayList<>();
try (PDDocument document = pdfDocumentFactory.load(multipartFile, true)) {
int pageCount = document.getNumberOfPages();
for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
String pageText = pdfContentExtractor.extractPageTextRaw(document, pageNumber);
if (pageText != null && !pageText.isBlank()) {
pages.add(new AiPageText(pageNumber, pageText));
pages.add(new AiRagPageText(pageNumber, pageText));
}
}
}
AiDocumentIngestRequest ingestRequest =
new AiDocumentIngestRequest(file.getId(), file.getName(), pages);
AiRagIngestRequest ingestRequest =
new AiRagIngestRequest(file.getId(), file.getName(), pages);
String body = objectMapper.writeValueAsString(ingestRequest);
aiEngineClient.postLongRunning(DOCUMENTS_ENDPOINT, body);
aiEngineClient.postLongRunning(RAG_DOCUMENTS_ENDPOINT, body);
log.debug(
"Ingested document: id={}, name={}, pages={}",
"Ingested file into RAG: id={}, name={}, pages={}",
file.getId(),
file.getName(),
pages.size());
@@ -365,29 +355,6 @@ public class AiWorkflowService {
return new WorkflowState.Terminal(response);
}
private WorkflowState onGenerateFile(AiWorkflowResponse response, ProgressListener listener)
throws IOException {
String content = response.getGeneratedContent();
String filename = response.getGeneratedFilename();
if (content == null || filename == null || filename.isBlank()) {
return new WorkflowState.Terminal(
cannotContinue(
"AI engine returned generate_file without content or filename."));
}
listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.PROCESSING));
String safeFilename = Filenames.toSimpleFileName(filename);
byte[] bytes = content.getBytes(java.nio.charset.StandardCharsets.UTF_8);
org.springframework.core.io.Resource resource =
new org.springframework.core.io.ByteArrayResource(bytes) {
@Override
public String getFilename() {
return safeFilename;
}
};
return new WorkflowState.Terminal(
buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null));
}
@SuppressWarnings("unchecked")
private WorkflowState runPlan(
List<Map<String, Object>> steps,
@@ -684,57 +651,10 @@ public class AiWorkflowService {
return response;
}
/**
* Drive the engine's streaming orchestrator endpoint. Progress events are forwarded to {@code
* listener} as they arrive (each one keeps the SSE connection to the frontend alive too). The
* final {@code result} event carries the full {@link AiWorkflowResponse}; an {@code error}
* event surfaces engine-side failures.
*/
private AiWorkflowResponse invokeOrchestrator(
WorkflowTurnRequest request, ProgressListener listener) throws IOException {
private AiWorkflowResponse invokeOrchestrator(WorkflowTurnRequest request) throws IOException {
String requestBody = objectMapper.writeValueAsString(request);
AiWorkflowResponse[] resultHolder = new AiWorkflowResponse[1];
String[] errorHolder = new String[1];
aiEngineClient.streamPost(
"/api/v1/orchestrator",
requestBody,
line -> handleStreamLine(line, listener, resultHolder, errorHolder));
if (errorHolder[0] != null) {
throw new IOException("AI engine returned error: " + errorHolder[0]);
}
if (resultHolder[0] == null) {
throw new IOException("AI engine stream ended without a result");
}
return resultHolder[0];
}
private void handleStreamLine(
String line,
ProgressListener listener,
AiWorkflowResponse[] resultHolder,
String[] errorHolder) {
try {
JsonNode node = objectMapper.readTree(line);
String event = node.path("event").asText();
switch (event) {
case "progress" -> {
AiEngineProgressDetail detail =
objectMapper.treeToValue(node, AiEngineProgressDetail.class);
listener.onProgress(AiWorkflowProgressEvent.engineProgress(detail));
}
case "result" -> {
JsonNode response = node.path("response");
resultHolder[0] = objectMapper.treeToValue(response, AiWorkflowResponse.class);
}
case "error" -> errorHolder[0] = node.path("message").asText("unknown error");
case "heartbeat" -> listener.onHeartbeat();
default -> log.warn("Ignoring unknown engine stream event: {}", event);
}
} catch (JacksonException e) {
log.warn("Failed to parse engine stream line: {}", line, e);
}
String responseBody = aiEngineClient.post("/api/v1/orchestrator", requestBody);
return objectMapper.readValue(responseBody, AiWorkflowResponse.class);
}
@Data
@@ -3,6 +3,7 @@ package stirling.software.proprietary.service;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -11,7 +12,6 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.QuoteMode;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
@@ -21,30 +21,25 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonValue;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.pdf.parser.PdfIngester;
import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.PdfUtils;
import stirling.software.proprietary.model.api.ai.AiPdfContentType;
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
import stirling.software.proprietary.model.api.ai.FolioType;
import stirling.software.proprietary.pdf.FlexibleCSVWriter;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.Table;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
@Slf4j
@Service
@RequiredArgsConstructor
public class PdfContentExtractor {
private final TabulaTableParser tabulaTableParser;
private final PdfIngester pdfIngester;
private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
private static final int TEXT_PRESENCE_THRESHOLD = 20;
@@ -106,21 +101,21 @@ public class PdfContentExtractor {
* @return list of CSV strings (one per table), empty if no tables found
*/
public List<String> extractTablesAsCsv(PDDocument document, int pageNumber) throws IOException {
List<TableFragment> fragments = tabulaTableParser.parse(document, pageNumber);
if (fragments.isEmpty()) return List.of();
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
CSVFormat format =
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
List<String> csvStrings = new ArrayList<>();
for (TableFragment fragment : fragments) {
StringWriter sw = new StringWriter();
try (CSVPrinter printer = format.print(sw)) {
for (List<String> row : fragment.rawRows()) {
printer.printRecord(row);
}
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
Page tabulaPage = extractor.extract(pageNumber);
List<Table> tables = sea.extract(tabulaPage);
for (Table table : tables) {
StringWriter sw = new StringWriter();
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
csvWriter.write(sw, Collections.singletonList(table));
csvStrings.add(sw.toString());
}
csvStrings.add(sw.toString());
}
return csvStrings;
}
@@ -188,8 +183,6 @@ public class PdfContentExtractor {
case PAGE_TEXT, FULL_TEXT ->
Optional.<PdfContentResult>ofNullable(
extractText(lf, fileReq, remainingPages, remainingCharacters));
case PAGE_LAYOUT ->
Optional.<PdfContentResult>ofNullable(extractPageLayout(lf, remainingPages));
default -> {
log.warn(
"Content type {} not yet implemented, skipping for {}",
@@ -214,35 +207,6 @@ public class PdfContentExtractor {
return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted);
}
private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException {
List<ParsedPage> parsedPages = pdfIngester.parse(lf.document(), maxPages);
List<LayoutPage> pages = new ArrayList<>();
for (ParsedPage pp : parsedPages) {
if (pp.layoutLines().isEmpty()) continue;
List<LayoutLine> lines = new ArrayList<>();
for (RawLine rawLine : pp.layoutLines()) {
List<LayoutFragment> fragments = new ArrayList<>();
for (TextFragment tf : rawLine.fragments()) {
fragments.add(
new LayoutFragment(
tf.text(),
tf.bounds().x(),
tf.bounds().y(),
tf.bounds().width(),
tf.fontSize(),
tf.bold()));
}
lines.add(new LayoutLine(rawLine.bounds().y(), fragments));
}
pages.add(new LayoutPage(pp.pageNumber(), lines));
}
if (pages.isEmpty()) return null;
PageLayoutFileResult result = new PageLayoutFileResult();
result.setFileName(lf.fileName());
result.setPages(pages);
return result;
}
private WorkflowArtifact buildArtifact(ArtifactKind kind, List<PdfContentResult> results) {
return switch (kind) {
case EXTRACTED_TEXT -> {
@@ -250,12 +214,10 @@ public class PdfContentExtractor {
artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList());
yield artifact;
}
case PAGE_LAYOUT -> {
PageLayoutArtifact artifact = new PageLayoutArtifact();
artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList());
yield artifact;
}
case TOOL_REPORT ->
// TOOL_REPORT artifacts don't come from PDF content extraction — they're
// built by AiWorkflowService from tool-response metadata. Never reached
// from this code path; presence in the enum is to satisfy the switch.
throw new IllegalArgumentException(
"TOOL_REPORT artifacts are not produced by PdfContentExtractor");
};
@@ -369,7 +331,6 @@ public class PdfContentExtractor {
*/
enum ArtifactKind {
EXTRACTED_TEXT("extracted_text"),
PAGE_LAYOUT("page_layout"),
TOOL_REPORT("tool_report");
private final String value;
@@ -433,40 +394,4 @@ public class PdfContentExtractor {
this.report = report;
}
}
// Serialization contract with the Python engine — see PageLayoutArtifactContractTest.
/** One text fragment with its bounding-box geometry and font properties. */
record LayoutFragment(
String text, float x, float y, float width, float fontSize, boolean bold) {}
/** A visual line on the page: y-coordinate and all fragments on that line. */
record LayoutLine(float y, List<LayoutFragment> fragments) {}
/** All layout lines for a single page. */
record LayoutPage(int pageNumber, List<LayoutLine> lines) {}
/** Page layout data for one file, as a PdfContentResult. */
@Data
static final class PageLayoutFileResult implements PdfContentResult {
private String fileName;
private List<LayoutPage> pages = new ArrayList<>();
@Override
public ArtifactKind getArtifactKind() {
return ArtifactKind.PAGE_LAYOUT;
}
@Override
public int pagesConsumed() {
return pages.size();
}
}
/** Artifact carrying full spatial page layout for all input files. */
@Data
static final class PageLayoutArtifact implements WorkflowArtifact {
private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT;
private List<PageLayoutFileResult> files = new ArrayList<>();
}
}
@@ -8,7 +8,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -23,7 +22,6 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -408,31 +406,6 @@ class AiWorkflowServiceTest {
verify(internalApiClient, times(1)).post(eq(COMPRESS_ENDPOINT), any());
}
@Test
void generateFileStoresContentDirectlyWithoutToolCall() throws IOException {
MockMultipartFile input = pdf("report.pdf", "bytes");
stubOrchestrator(
"""
{
"outcome":"generate_file",
"content":"# Hello\\n\\nWorld",
"filename":"report-reconstruction.md",
"summary":"Reconstructed the document as a Markdown file."
}
""");
AtomicInteger ids = stubFileStorage();
AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown"));
assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome());
assertEquals(1, result.getResultFiles().size());
assertEquals("report-reconstruction.md", result.getResultFiles().get(0).getFileName());
assertEquals("file-1", result.getResultFiles().get(0).getFileId());
assertEquals(1, ids.get());
// No tool endpoint should be called — content goes directly to file storage.
verify(internalApiClient, never()).post(anyString(), any());
}
@Test
void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException {
MockMultipartFile input = pdf("input.pdf", "bytes");
@@ -446,7 +419,7 @@ class AiWorkflowServiceTest {
}
@Test
void needIngestExtractsPageTextAndPostsThenRetries() throws IOException {
void needIngestExtractsPageTextAndPostsToRagThenRetries() throws IOException {
MockMultipartFile input = pdf("report.pdf", "bytes");
when(fileIdStrategy.idFor(any())).thenReturn("report-id");
@@ -458,64 +431,37 @@ class AiWorkflowServiceTest {
.thenReturn("page content");
int[] orchestratorCalls = {0};
doAnswer(
when(aiEngineClient.post(eq("/api/v1/orchestrator"), anyString()))
.thenAnswer(
inv -> {
orchestratorCalls[0]++;
String responseJson;
if (orchestratorCalls[0] == 1) {
responseJson =
"""
{
"outcome":"need_ingest",
"resumeWith":"pdf_question",
"reason":"ingest first",
"filesToIngest":[{"id":"report-id","name":"report.pdf"}],
"contentTypes":["page_text"]
}
""";
} else {
responseJson =
"""
{"outcome":"answer","answer":"done","evidence":[]}
""";
return """
{
"outcome":"need_ingest",
"resumeWith":"pdf_question",
"reason":"ingest first",
"filesToIngest":[{"id":"report-id","name":"report.pdf"}],
"contentTypes":["page_text"]
}
""";
}
Consumer<String> consumer = inv.getArgument(2);
consumer.accept(wrapAsResultEvent(responseJson));
return null;
})
.when(aiEngineClient)
.streamPost(eq("/api/v1/orchestrator"), anyString(), any());
return """
{"outcome":"answer","answer":"done","evidence":[]}
""";
});
AiWorkflowResponse result = service.orchestrate(requestFor(input, "summarise this"));
assertEquals(AiWorkflowOutcome.ANSWER, result.getOutcome());
verify(aiEngineClient, times(1)).postLongRunning(eq("/api/v1/documents"), anyString());
verify(aiEngineClient, times(2)).streamPost(eq("/api/v1/orchestrator"), anyString(), any());
verify(aiEngineClient, times(1)).postLongRunning(eq("/api/v1/rag/documents"), anyString());
verify(aiEngineClient, times(2)).post(eq("/api/v1/orchestrator"), anyString());
}
// --- helpers ---
private void stubOrchestrator(String responseJson) throws IOException {
doAnswer(
inv -> {
Consumer<String> consumer = inv.getArgument(2);
consumer.accept(wrapAsResultEvent(responseJson));
return null;
})
.when(aiEngineClient)
.streamPost(eq("/api/v1/orchestrator"), anyString(), any());
}
/**
* Wrap a unary orchestrator response JSON as the NDJSON "result" event the streaming endpoint
* emits, so existing tests can keep their per-outcome JSON literals.
*/
private String wrapAsResultEvent(String responseJson) throws IOException {
return objectMapper
.createObjectNode()
.put("event", "result")
.set("response", objectMapper.readTree(responseJson))
.toString();
when(aiEngineClient.post(eq("/api/v1/orchestrator"), anyString())).thenReturn(responseJson);
}
private void stubEndpoint(String endpoint, Resource body) {
@@ -1,66 +0,0 @@
package stirling.software.proprietary.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.junit.jupiter.api.Test;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage;
import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact;
import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.json.JsonMapper;
/**
* Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that
* the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}.
*
* <p>The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON
* literal and asserts field values. If either side renames a field, one of these tests fails.
*/
class PageLayoutArtifactContractTest {
static final String CONTRACT_JSON =
"""
{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}""";
@Test
void pageLayoutArtifact_serialisesToExpectedJson() throws Exception {
LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true);
LayoutLine line = new LayoutLine(10.0f, List.of(fragment));
LayoutPage page = new LayoutPage(1, List.of(line));
PageLayoutFileResult fileResult = new PageLayoutFileResult();
fileResult.setFileName("test.pdf");
fileResult.setPages(List.of(page));
PageLayoutArtifact artifact = new PageLayoutArtifact();
artifact.setFiles(List.of(fileResult));
JsonNode json = new JsonMapper().valueToTree(artifact);
assertEquals("page_layout", json.get("kind").asText());
JsonNode file = json.get("files").get(0);
assertEquals("test.pdf", file.get("fileName").asText());
JsonNode pg = file.get("pages").get(0);
assertEquals(1, pg.get("pageNumber").asInt());
JsonNode ln = pg.get("lines").get(0);
assertEquals(10.0, ln.get("y").asDouble(), 0.001);
JsonNode frag = ln.get("fragments").get(0);
assertEquals("Hello", frag.get("text").asText());
assertEquals(1.0, frag.get("x").asDouble(), 0.001);
assertEquals(2.0, frag.get("y").asDouble(), 0.001);
assertEquals(30.0, frag.get("width").asDouble(), 0.001);
assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001);
assertTrue(frag.get("bold").asBoolean());
}
}
+5 -8
View File
@@ -101,14 +101,11 @@ tasks.register('syncAppVersion') {
def tauriConfigFile = new File(tauriConfigPath)
if (tauriConfigFile.exists()) {
def content = tauriConfigFile.getText('UTF-8')
def matcher = (content =~ /(?m)^(\s*"version":\s*")([^"]*)(")/)
if (!matcher.find()) {
throw new GradleException("Could not locate version in ${tauriConfigFile} for synchronization")
}
def updatedContent = matcher.replaceFirst("${matcher.group(1)}${appVersionStr}${matcher.group(3)}")
if (content != updatedContent) {
tauriConfigFile.write(updatedContent, 'UTF-8')
def parsed = new groovy.json.JsonSlurper().parse(tauriConfigFile)
parsed.version = appVersionStr
def formatted = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(parsed)) + System.lineSeparator()
if (tauriConfigFile.getText('UTF-8') != formatted) {
tauriConfigFile.write(formatted, 'UTF-8')
}
}
View File
-21
View File
@@ -37,22 +37,6 @@ STIRLING_RAG_TOP_K=20
# rather than chain more searches.
STIRLING_RAG_MAX_SEARCHES=5
# Chunked reasoner settings: how big each per-worker slice is (in characters),
# how many workers may run in parallel against the fast model, and how long
# any single worker is allowed to wait for a response before being abandoned.
# Worker timeouts protect gather_notes from upstream model stalls (which
# otherwise hang at the provider's ~10 minute HTTP default); the affected
# slice is dropped and the rest of the document still answers.
STIRLING_CHUNKED_REASONER_CHARS_PER_SLICE=16000
STIRLING_CHUNKED_REASONER_CONCURRENCY=10
STIRLING_CHUNKED_REASONER_WORKER_TIMEOUT_SECONDS=60
# When the rendered slice notes would exceed this many characters, the
# reasoner folds them hierarchically with fast-model calls until they fit.
# This keeps the synthesis prompt under the model's context limit on long
# documents (a 3000-page novel produces ~900k chars of raw notes).
STIRLING_CHUNKED_REASONER_NOTES_CHAR_BUDGET=250000
# Upper bounds on PDF page text the engine will request per extraction round.
STIRLING_MAX_PAGES=200
STIRLING_MAX_CHARACTERS=200000
@@ -67,8 +51,3 @@ STIRLING_LOG_LEVEL=INFO
# Path to log file. Rolls daily, keeps 1 backup. Leave empty for console only.
STIRLING_LOG_FILE=
# Set true to log every outgoing httpx / Anthropic SDK request with timing.
# Use when diagnosing worker stalls: a hung call shows a "Request" line with
# no matching "Response" line. Noisy; leave off in normal use.
STIRLING_HTTP_DEBUG=false
+5 -22
View File
@@ -21,7 +21,7 @@ from datamodel_code_generator.format import Formatter
from referencing import Registry, Resource
from referencing.jsonschema import DRAFT202012
# Fields inherited from PDFFile base class - not tool parameters.
# Fields inherited from PDFFile base class — not tool parameters.
BASE_CLASS_FIELDS = frozenset({"fileInput", "fileId"})
_ENGINE_ROOT = Path(__file__).resolve().parents[1]
@@ -73,9 +73,8 @@ class ToolDiscovery:
for path, path_item in sorted(self.spec.get("paths", {}).items()):
if "{" in path or not any(path.startswith(p) for p in self.ALLOWED_PATH_PREFIXES):
continue
body_schema = self._get_request_body_schema(path_item) or {}
body_props = self._get_request_properties(path_item) or {}
query_props = self._get_query_parameters(path_item)
body_props = body_schema.get("properties") or {}
# Body properties win on name collision — body is the canonical param source
# for the existing tools; query params are additive.
properties = {**query_props, **body_props}
@@ -88,21 +87,7 @@ class ToolDiscovery:
enum_name = _deduplicate(_path_to_enum_name(path), used_enum)
class_name = _deduplicate(_path_to_class_name(path), used_class)
entry: dict[str, Any] = {
"type": "object",
"properties": clean_props,
"description": body_schema.get("description"),
}
# Calculate which fields are actually required (many are marked as required,
# but have a default set, so they're not really required)
required = [
name
for name in body_schema.get("required") or []
if name in clean_props and "default" not in (clean_props[name] or {})
]
if required:
entry["required"] = required
defs[class_name] = entry
defs[class_name] = {"type": "object", "properties": clean_props}
tools.append(ToolSpec(path, enum_name, class_name))
self._inline_component_refs(defs)
@@ -134,7 +119,7 @@ class ToolDiscovery:
return self.resolver.lookup(schema["$ref"]).contents
return schema
def _get_request_body_schema(self, path_item: dict[str, Any]) -> dict[str, Any] | None:
def _get_request_properties(self, path_item: dict[str, Any]) -> dict[str, Any] | None:
post = path_item.get("post")
if not post:
return None
@@ -143,7 +128,7 @@ class ToolDiscovery:
if media_type in content:
schema = content[media_type].get("schema")
if schema:
return self._resolve_ref(schema)
return self._resolve_ref(schema).get("properties")
return None
def _get_query_parameters(self, path_item: dict[str, Any]) -> dict[str, Any]:
@@ -242,8 +227,6 @@ def generate_models_code(combined_schema: dict[str, Any]) -> str:
field_constraints=True,
no_alias=True,
set_default_enum_member=True,
strict_nullable=True,
use_schema_description=True,
additional_imports=["enum.StrEnum"],
enable_version_header=False,
custom_file_header=_FILE_HEADER,
-2
View File
@@ -5,7 +5,6 @@ from .orchestrator import OrchestratorAgent
from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection
from .pdf_questions import PdfQuestionAgent
from .pdf_review import PdfReviewAgent
from .pdf_to_markdown import PdfToMarkdownAgent
from .user_spec import UserSpecAgent
__all__ = [
@@ -16,6 +15,5 @@ __all__ = [
"PdfEditPlanSelection",
"PdfQuestionAgent",
"PdfReviewAgent",
"PdfToMarkdownAgent",
"UserSpecAgent",
]
@@ -11,14 +11,12 @@ from pydantic_ai.tools import RunContext
from stirling.agents.pdf_edit import PdfEditAgent
from stirling.agents.pdf_questions import PdfQuestionAgent
from stirling.agents.pdf_review import PdfReviewAgent
from stirling.agents.pdf_to_markdown import PdfToMarkdownAgent
from stirling.agents.user_spec import UserSpecAgent
from stirling.contracts import (
AgentDraftWorkflowResponse,
ExtractedTextArtifact,
OrchestratorRequest,
OrchestratorResponse,
PageLayoutArtifact,
PdfEditResponse,
PdfQuestionOrchestrateResponse,
SupportedCapability,
@@ -27,7 +25,6 @@ from stirling.contracts import (
format_file_names,
)
from stirling.contracts.pdf_edit import EditPlanResponse
from stirling.contracts.pdf_to_markdown import PdfToMarkdownOrchestrateResponse
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
@@ -71,11 +68,6 @@ class OrchestratorAgent:
" feedback')."
),
),
ToolOutput(
self.delegate_pdf_to_markdown,
name="delegate_pdf_to_markdown",
description=("Delegate requests to reconstruct a PDF as a Markdown document."),
),
ToolOutput(
self.unsupported_capability,
name="unsupported_capability",
@@ -92,8 +84,6 @@ class OrchestratorAgent:
"Use delegate_pdf_review when the user wants the PDF returned with review"
" comments attached — anything like 'review this', 'annotate with comments',"
" 'leave feedback on the PDF'. "
"Use delegate_pdf_to_markdown for any request to convert a PDF to Markdown "
"or reconstruct its content as readable text. "
"Use unsupported_capability when the user asks about the assistant itself "
"or when none of the other outputs fit; supply a helpful message."
),
@@ -133,8 +123,6 @@ class OrchestratorAgent:
return await self._run_pdf_edit(request)
case SupportedCapability.AGENT_DRAFT:
return await self._run_agent_draft(request)
case SupportedCapability.PDF_TO_MARKDOWN:
return await self._run_pdf_to_markdown(request)
case (
SupportedCapability.ORCHESTRATE
| SupportedCapability.AGENT_REVISE
@@ -163,12 +151,6 @@ class OrchestratorAgent:
async def _run_agent_draft(self, request: OrchestratorRequest) -> AgentDraftWorkflowResponse:
return await UserSpecAgent(self.runtime).orchestrate(request)
async def delegate_pdf_to_markdown(self, ctx: RunContext[OrchestratorDeps]) -> PdfToMarkdownOrchestrateResponse:
return await self._run_pdf_to_markdown(ctx.deps.request)
async def _run_pdf_to_markdown(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse:
return await PdfToMarkdownAgent(self.runtime).orchestrate(request)
async def delegate_pdf_review(self, ctx: RunContext[OrchestratorDeps]) -> EditPlanResponse:
return await self._run_pdf_review(ctx.deps.request)
@@ -204,10 +186,5 @@ class OrchestratorAgent:
file_names = [f.file_name for f in artifact.files]
descriptions.append(f"- extracted_text: {total_pages} pages from {file_names}")
continue
if isinstance(artifact, PageLayoutArtifact):
total_pages = sum(len(f.pages) for f in artifact.files)
file_names = [f.file_name for f in artifact.files]
descriptions.append(f"- page_layout: {total_pages} pages from {file_names}")
continue
descriptions.append("- unknown artifact")
return "\n".join(descriptions)
+3 -32
View File
@@ -34,9 +34,9 @@ logger = logging.getLogger(__name__)
class PdfEditPlanSelection(ApiModel):
outcome: Literal["plan"] = "plan"
rationale: str
operations: list[ToolEndpoint] = Field(min_length=1)
summary: str
rationale: str | None = None
type PdfEditPlanOutput = PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | NeedContentResponse
@@ -254,16 +254,10 @@ class PdfEditAgent:
"Plan PDF edit requests. "
f"Supported operations are: {self._get_operations_prompt(supported_operations)}."
f"{unavailable_clause} "
"Each operation in the user-facing prompt is listed with its full set of parameters. "
"Treat that list as authoritative: an operation can ONLY do what its listed parameters "
"and description allow. "
"Return an ordered list of one or more supported operations for the plan. "
"Chain multiple operations together whenever the request needs effects that no single "
"supported operation provides on its own (for example, splitting then rotating then "
"merging, or extracting pages then re-inserting them). "
"Only return cannot_do when no sequence of the supported operations could achieve the request. "
"Do not produce operation parameters in this stage. "
"Return need_clarification when the request is genuinely ambiguous. "
"Return cannot_do when the request is outside the supported operations. "
"Return plan when a reasonable multi-step plan can be created. "
"Never return partial plans."
),
@@ -286,7 +280,7 @@ class PdfEditAgent:
f"Conversation history:\n{format_conversation_history(request.conversation_history)}\n"
f"User request: {request.user_message}\n"
f"Files: {format_file_names(request.files)}\n"
f"Supported operations:\n{self._get_supported_operations_prompt(supported_operations)}\n"
f"Supported operations: {self._get_operations_prompt(supported_operations)}\n"
f"{unavailable_line}"
f"Extracted page text:\n{format_page_text(request.page_text)}"
)
@@ -303,29 +297,6 @@ class PdfEditAgent:
def _get_operations_prompt(operations: Iterable[ToolEndpoint]) -> str:
return ", ".join(f"{op.name} ({op.value})" for op in operations)
@staticmethod
def _get_supported_operations_prompt(operations: Iterable[ToolEndpoint]) -> str:
"""Render each operation with its description and a flat list of param descriptions.
The selection step decides which tool fits the user's request, so it just needs
what each tool does and what knobs it has, not the full schema.
"""
lines: list[str] = []
for op in operations:
schema = OPERATIONS[op].model_json_schema()
head = f"- {op.name} ({op.value})"
description = (schema.get("description") or "").strip()
if description:
head += f": {description}"
lines.append(head)
for name, prop in (schema.get("properties") or {}).items():
param_description = (prop.get("description") or "").strip()
if param_description:
lines.append(f" {name}: {param_description}")
else:
lines.append(f" {name}")
return "\n".join(lines)
def _fill_need_content_defaults(
self,
selection: NeedContentResponse,
+25 -45
View File
@@ -6,7 +6,6 @@ from pydantic_ai import Agent
from pydantic_ai.output import NativeOutput
from stirling.agents.math_presentation import MathIntentClassifier, extract_math_verdict
from stirling.agents.shared import ChunkedReasoner, WholeDocReaderCapability
from stirling.contracts import (
AiFile,
EditPlanResponse,
@@ -25,46 +24,44 @@ from stirling.contracts import (
format_conversation_history,
format_file_names,
)
from stirling.documents import RagCapability
from stirling.models.agent_tool_models import AgentToolId, MathAuditorAgentParams
from stirling.rag import RagCapability
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
PDF_QUESTION_SYSTEM_PROMPT = (
"You answer questions about PDF documents using two retrieval tools:\n"
"You answer questions about PDF documents by retrieving relevant content with the "
"search_knowledge tool. Use it before answering. Do not guess or use outside knowledge.\n"
"\n"
"1. search_knowledge(query) - returns the passages most semantically similar "
"to the query. Use it for targeted lookups: a specific fact, a named section, "
"a particular passage. Typically one or two calls is enough.\n"
"\n"
"2. read_full_document(query) - reads every page of the attached documents in "
"parallel and returns notes relevant to the query. Use it when answering "
"requires seeing the whole document end-to-end: summaries, aggregations "
"(largest, shortest, count), comparisons across sections. It is more "
"expensive than search_knowledge, so prefer search_knowledge when one or two "
"passages would suffice.\n"
"\n"
"Pick the right tool, call it, then answer from what you got back. Do not "
"guess or use outside knowledge.\n"
"The search_knowledge tool has a finite call budget per run. When it is no longer "
"available, answer from what you have already retrieved.\n"
"\n"
"Guidelines:\n"
"- If the retrieved content does not support a confident answer, return not_found.\n"
"- Include a short list of evidence snippets (with page numbers where available) "
"drawn from what the tools returned.\n"
"- Make targeted search_knowledge calls. Typically one or two is enough.\n"
"- Answer from the retrieved text. If the retrieved content doesn't support a confident "
"answer, return not_found.\n"
"- For questions that would require reading the entire document end-to-end (e.g. "
"'what's the shortest chapter', 'how many X are there'), return not_found.\n"
"- Include a short list of evidence snippets (with page numbers where available) drawn "
"from what search_knowledge returned.\n"
"\n"
"Writing the not_found reason:\n"
"- The reason is shown directly to the end user, so write it in plain, friendly "
"language. One or two short sentences.\n"
"- NEVER mention 'RAG', 'retrieval', 'chunks', 'search results', 'targeted search', "
"'search_knowledge', 'read_full_document', or other implementation details.\n"
"'search_knowledge', or other implementation details.\n"
"- Be honest about the actual limitation. For questions that require full-document "
"analysis (shortest chapter, word counts, etc.), explain that the document is too "
"long to analyse end-to-end: you can only look up specific passages, and that's "
"not enough to compare every part of the document against every other.\n"
"- For questions where the answer just isn't in the document, say so directly: "
"'I couldn't find that information in the document.'\n"
"- Do not make it sound like you're choosing not to answer."
"- Do not make it sound like you're choosing not to answer. Be clear that it's "
"a genuine constraint."
)
_MATH_SYNTH_SYSTEM_PROMPT = (
"You are given a math-audit Verdict (structured JSON) and the user's "
"original question. Answer the question in plain prose using only "
@@ -86,9 +83,6 @@ class PdfQuestionAgent:
model_settings=runtime.fast_model_settings,
)
self._math_intent_classifier = MathIntentClassifier(runtime)
# Shared across whole-doc-reader instances so the worker agent and
# semaphore are constructed once and reused per request.
self._chunked_reasoner = ChunkedReasoner(runtime)
async def handle(self, request: PdfQuestionRequest) -> PdfQuestionResponse:
logger.info(
@@ -101,7 +95,7 @@ class PdfQuestionAgent:
logger.info("[pdf-question] missing ingestions: %s", [file.name for file in missing])
return NeedIngestResponse(
resume_with=SupportedCapability.PDF_QUESTION,
reason="Some files have not been ingested yet.",
reason="Some files have not been ingested into RAG yet.",
files_to_ingest=missing,
content_types=[PdfContentType.PAGE_TEXT],
)
@@ -151,37 +145,23 @@ class PdfQuestionAgent:
async def _find_missing_files(self, files: list[AiFile]) -> list[AiFile]:
missing: list[AiFile] = []
for file in files:
if not await self.runtime.documents.has_collection(file.id):
if not await self.runtime.rag_service.has_collection(file.id):
missing.append(file)
return missing
async def _run_answer_agent(self, request: PdfQuestionRequest) -> PdfQuestionTerminalResponse:
"""Drive a single smart-model agent with both retrieval tools.
The agent picks ``search_knowledge`` for targeted lookups and
``read_full_document`` for whole-document questions. Removing the
upstream classifier keeps that judgement in the same call that writes
the answer, and lets the agent mix tools when the question warrants it.
"""
rag = RagCapability(
documents=self.runtime.documents,
rag_service=self.runtime.rag_service,
collections=[file.id for file in request.files],
top_k=self.runtime.settings.rag_default_top_k,
max_searches=self.runtime.settings.rag_max_searches,
)
whole_doc = WholeDocReaderCapability(
runtime=self.runtime,
files=request.files,
reasoner=self._chunked_reasoner,
)
agent = Agent(
model=self.runtime.smart_model,
output_type=NativeOutput([PdfQuestionAnswerResponse, PdfQuestionNotFoundResponse]),
system_prompt=PDF_QUESTION_SYSTEM_PROMPT,
# pydantic-ai accepts a list of (string-or-callable) instruction sources;
# it resolves each at run time and concatenates them for the model.
instructions=[rag.instructions, whole_doc.instructions],
toolsets=[rag.toolset, whole_doc.toolset],
instructions=rag.instructions,
toolsets=[rag.toolset],
model_settings=self.runtime.smart_model_settings,
)
prompt = self._build_prompt(request)
@@ -204,5 +184,5 @@ class PdfQuestionAgent:
f"Conversation history:\n{history}\n"
f"Files: {format_file_names(request.files)}\n"
f"Question: {request.question}\n"
"Pick the right retrieval tool for this question, then answer from what it returns."
"Use search_knowledge to retrieve the relevant content, then answer."
)
@@ -1,3 +0,0 @@
from .agent import PdfToMarkdownAgent
__all__ = ["PdfToMarkdownAgent"]
@@ -1,435 +0,0 @@
"""PDF to Markdown Agent.
Converts a parsed PDF document into a single clean Markdown document, preserving
headings, paragraphs, and tables in reading order.
"""
from __future__ import annotations
import asyncio
import logging
import re
import time
from pydantic import BaseModel, Field
from pydantic_ai import Agent
from pydantic_ai.output import NativeOutput
from stirling.contracts import (
EditCannotDoResponse,
GenerateFileResponse,
NeedContentFileRequest,
NeedContentResponse,
OrchestratorRequest,
PdfContentType,
SupportedCapability,
format_conversation_history,
)
from stirling.contracts.pdf_to_markdown import (
PageLayout,
PageLayoutArtifact,
PdfToMarkdownCannotDoResponse,
PdfToMarkdownOrchestrateResponse,
PdfToMarkdownRequest,
PdfToMarkdownResponse,
PdfToMarkdownSuccessResponse,
)
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
# Warn when output tokens are close to the typical model output limit (~8192 for most
# configurations). The actual limit is model-specific; this threshold catches likely truncation.
_OUTPUT_TOKEN_TRUNCATION_THRESHOLD = 7500
# Chunking limits — keep each LLM call to a manageable payload size.
# Fragment count is the primary driver of JSON payload size (each fragment carries x/y/width/
# fontSize/bold metadata beyond its text). Page cap prevents low-text pages accumulating.
_MAX_CHUNK_FRAGMENTS = 1_000
_MAX_CHUNK_PAGES = 10
# Max concurrent LLM calls — limits API rate pressure on large documents.
_MAX_PARALLEL_CHUNKS = 3
# ── LLM output model ────────────────────────────────────────────────────────────────────────────
class _ReconstructionOutput(BaseModel):
markdown: str = Field(description="Full document reconstructed as clean Markdown.")
# ── Agent ────────────────────────────────────────────────────────────────────────────────────────
class PdfToMarkdownAgent:
def __init__(self, runtime: AppRuntime) -> None:
self.runtime = runtime
self._sem = asyncio.Semaphore(_MAX_PARALLEL_CHUNKS)
self._reconstruct_agent = Agent(
model=runtime.smart_model,
output_type=NativeOutput(_ReconstructionOutput),
system_prompt=(
"You reconstruct PDF pages into clean Markdown from spatial fragment data.\n"
"Input: PAGE LAYOUT — per-fragment x/y/font data for structural analysis.\n\n"
"COLUMN DETECTION (for tables in page_layout):\n"
"- Look at the x-positions of fragments across 3+ consecutive lines.\n"
"- If fragments cluster at the same x-positions across multiple lines, those are table columns.\n"
"- Each distinct x-cluster is one column."
" Name them from the header row (the first line in the cluster).\n"
"- Do NOT merge values from different x-columns into one cell.\n\n"
"ROW DETECTION:\n"
"- Each unique y-coordinate (or group within 3pt) is one table row.\n"
"- Every line of layout data is its own row — do not merge rows.\n"
"- If a column has no fragment on a given y-row, that cell is empty.\n\n"
"TABLE RENDERING:\n"
"- Render as: | col1 | col2 | col3 |\n"
" | --- | --- | --- |\n"
" | val | val | val |\n"
"- One source row = one table row. Never collapse multiple rows into one.\n"
"- Preserve numeric values exactly (no rounding, no formatting changes).\n"
"- Bold cells: wrap with ** in the Markdown cell.\n"
"- CRITICAL: the separator row `| --- | --- |` appears EXACTLY ONCE per table, immediately\n"
" after the header row. NEVER put `| --- |` after a data row or between data rows.\n"
" NEVER put a blank line inside a table. All rows (header + data) must be consecutive.\n"
"- Do NOT produce a header-only table followed by a second table with the data rows.\n"
" One logical table = one markdown table block, with header, one separator, then all data.\n\n"
"GROUP HEADERS (label-only rows inside a table):\n"
"- A row is a group header when: the first column has text AND every numeric column is empty.\n"
"- Do NOT render group headers as table rows with empty cells.\n"
"- Break the table, emit the label as **bold text** on its own line,"
" then start a new table for the rows that follow.\n"
"- Example labels: 'Policy functions', 'Non-current assets'.\n\n"
"TOTAL AND SUBTOTAL ROWS:\n"
"- Detect rows whose first cell contains (case-insensitive):"
" total, subtotal, surplus, balance, net, sum.\n"
"- These rows have numeric content — they are NOT group headers.\n"
"- Render the entire row in bold: | **Total income** | **1,234** | **5,678** |\n"
"- Keep total rows attached to the group they summarise.\n\n"
"MULTI-LEVEL TABLES (year or period as a row label):\n"
"- Detect when a row contains only a single label (a year like '2010' or period like 'Q1 2023')"
" with no numeric content, followed by repeated metric rows.\n"
"- Do NOT render the year as a table row.\n"
"- Normalise: add 'Year' as the first column, 'Metric' as the second,"
" and repeat the year value on each metric row.\n\n"
"PROSE REGIONS:\n"
"- Lines where x-positions vary across lines (not repeating columns) are prose.\n"
"- Merge lines at the same x-level into paragraphs. Separate indented lines.\n\n"
"HEADINGS:\n"
"- A line is a heading when it is bold OR font_size ≥2pt above body.\n"
" CRITICAL EXCEPTION: a bold fragment is a TABLE HEADER CELL, not a document heading, when\n"
" the same y-row in page_layout contains other fragments at different x-positions.\n"
" Only classify a bold line as a document heading when it is the SOLE fragment on its y-row.\n"
" Example: 'Non-current assets' at y=120 with '2010'@x=350, '2009'@x=420, '2008'@x=490\n"
" → this is a table header row, NOT a heading. Render it as the first cell of the table.\n"
"- Use ## for section headings, ### for sub-headings. Use # only for the document title.\n\n"
"ORDERING:\n"
"- Process content top-to-bottom as it appears on the page.\n"
"- Interleave prose blocks and table blocks in page order.\n"
"- Do not move text that appears before a table to after it, or vice versa.\n\n"
"FIDELITY:\n"
"- Do NOT invent, summarise, or omit any content.\n"
"- Do NOT add commentary, metadata, or JSON — output Markdown only."
),
model_settings={
**runtime.smart_model_settings,
"temperature": 0.0,
"max_tokens": _OUTPUT_TOKEN_TRUNCATION_THRESHOLD,
},
)
async def orchestrate(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse:
"""Entry point for the orchestrator delegate.
First turn: requests PAGE_LAYOUT extraction from Java via NeedContentResponse.
Resume turn: runs the LLM reconstruction and returns a write-file plan step.
"""
layout_artifact = next(
(a for a in request.artifacts if isinstance(a, PageLayoutArtifact)),
None,
)
if layout_artifact is None:
return NeedContentResponse(
resume_with=SupportedCapability.PDF_TO_MARKDOWN,
reason="Page layout data is required to reconstruct the document.",
files=[
NeedContentFileRequest(file=f, content_types=[PdfContentType.PAGE_LAYOUT]) for f in request.files
],
max_pages=self.runtime.settings.max_pages,
max_characters=self.runtime.settings.max_characters,
)
page_layout = [page for entry in layout_artifact.files for page in entry.pages]
file_names = [f.name for f in request.files]
result = await self.handle(
PdfToMarkdownRequest(
user_message=request.user_message,
file_names=file_names,
conversation_history=request.conversation_history,
page_layout=page_layout,
)
)
if isinstance(result, PdfToMarkdownCannotDoResponse):
return EditCannotDoResponse(reason=result.reason)
base = file_names[0].rsplit(".", 1)[0] if file_names else "document"
return GenerateFileResponse(
content=result.markdown,
filename=f"{base}-reconstruction.md",
summary="Reconstructed the document as a Markdown file.",
)
async def handle(self, request: PdfToMarkdownRequest) -> PdfToMarkdownResponse:
total_fragments = sum(len(line.fragments) for page in request.page_layout for line in page.lines)
logger.info(
"[pdf-to-markdown] received layout-pages=%d fragments=%d",
len(request.page_layout),
total_fragments,
)
if not request.page_layout:
logger.warning("[pdf-to-markdown] no content extracted from document; returning cannot_do")
return PdfToMarkdownCannotDoResponse(
reason=(
"No content was extracted from the document. "
"The file may be a scanned image PDF with no readable text. "
"Try running OCR on the document first."
)
)
chunks = _build_page_chunks(request.page_layout)
logger.info("[pdf-to-markdown] chunks=%d (max %d in parallel)", len(chunks), _MAX_PARALLEL_CHUNKS)
if len(chunks) == 1:
return await self._reconstruct_chunk(request, chunks[0], chunk_num=1, total_chunks=1)
total = len(chunks)
results = await asyncio.gather(
*(
self._reconstruct_chunk(request, chunk, chunk_num=i + 1, total_chunks=total)
for i, chunk in enumerate(chunks)
)
)
markdown_parts: list[str] = []
for result in results:
if isinstance(result, PdfToMarkdownSuccessResponse) and result.markdown:
markdown_parts.append(result.markdown)
elif isinstance(result, PdfToMarkdownCannotDoResponse):
logger.warning("[pdf-to-markdown] chunk dropped: %s", result.reason)
if not markdown_parts:
return PdfToMarkdownCannotDoResponse(reason="The document could not be reconstructed. All chunks failed.")
logger.info("[pdf-to-markdown] assembly: %d/%d chunks produced output", len(markdown_parts), len(chunks))
return PdfToMarkdownSuccessResponse(markdown="\n\n".join(markdown_parts))
async def _reconstruct_chunk(
self,
request: PdfToMarkdownRequest,
pages: list[PageLayout],
chunk_num: int,
total_chunks: int,
) -> PdfToMarkdownResponse:
chunk_request = PdfToMarkdownRequest(
user_message=request.user_message,
file_names=request.file_names,
conversation_history=request.conversation_history,
page_layout=pages,
)
try:
async with self._sem:
return await self._reconstruct_document(chunk_request, chunk_num, total_chunks)
except Exception as e:
logger.error("[pdf-to-markdown] chunk %d/%d failed: %s", chunk_num, total_chunks, e, exc_info=True)
return PdfToMarkdownCannotDoResponse(
reason="The document could not be reconstructed. The AI model failed to process it."
)
async def _reconstruct_document(
self, request: PdfToMarkdownRequest, chunk_num: int = 1, total_chunks: int = 1
) -> PdfToMarkdownSuccessResponse:
content = _build_reconstruction_prompt(request)
logger.info("[timing] chunk %d/%d llm-call prompt-chars=%d", chunk_num, total_chunks, len(content))
t0 = time.monotonic()
result = await self._reconstruct_agent.run([content])
llm_ms = int((time.monotonic() - t0) * 1000)
output: _ReconstructionOutput = result.output
usage = result.usage()
logger.info(
"[timing] chunk %d/%d llm-done ms=%d input-tokens=%s output-tokens=%s markdown-chars=%d",
chunk_num,
total_chunks,
llm_ms,
usage.input_tokens,
usage.output_tokens,
len(output.markdown),
)
if usage.output_tokens and usage.output_tokens >= _OUTPUT_TOKEN_TRUNCATION_THRESHOLD:
logger.warning(
"[timing] chunk %d/%d output likely truncated (output-tokens=%d)",
chunk_num,
total_chunks,
usage.output_tokens,
)
markdown = _remove_extra_separators(_fix_markdown_tables(_merge_orphaned_table_rows(output.markdown)))
return PdfToMarkdownSuccessResponse(markdown=markdown)
# ── Chunking ────────────────────────────────────────────────────────────────────────────────────
def _build_page_chunks(pages: list[PageLayout]) -> list[list[PageLayout]]:
chunks: list[list[PageLayout]] = []
current: list[PageLayout] = []
current_fragments = 0
for page in pages:
page_fragments = sum(len(line.fragments) for line in page.lines)
fragment_full = current and current_fragments + page_fragments > _MAX_CHUNK_FRAGMENTS
page_full = len(current) >= _MAX_CHUNK_PAGES
if fragment_full or page_full:
chunks.append(current)
current = []
current_fragments = 0
current.append(page)
current_fragments += page_fragments
if current:
chunks.append(current)
return chunks
# ── Prompt builders (module-level, no state) ────────────────────────────────────────────────────
def _build_reconstruction_prompt(request: PdfToMarkdownRequest) -> str:
history = format_conversation_history(request.conversation_history)
file_names = ", ".join(request.file_names) if request.file_names else "Unknown files"
layout_section = _format_layout(request.page_layout)
return (
f"Files: {file_names}\n\n"
f"User request: {request.user_message}\n\n"
f"Conversation history:\n{history}\n\n"
"PAGE LAYOUT (structural source — x/y fragment positions):\n"
"Each line is: y=NNN | text@(x,y) fs=N text@(x,y) fs=N ...\n"
"- y=NNN is the vertical position (row). Lines close in y are the same visual row.\n"
"- x=NNN is the horizontal position (column). Consistent x across rows = a column.\n"
"- fs=N is font size. Larger = likely a heading.\n"
"- **bold** markers indicate bold text.\n\n"
f"{layout_section}"
)
# ── LLM output post-processing ──────────────────────────────────────────────────────────────────
def _fix_markdown_tables(markdown: str) -> str:
"""Remove blank lines between table rows produced by the LLM."""
lines = markdown.split("\n")
result: list[str] = []
i = 0
while i < len(lines):
result.append(lines[i])
if lines[i].strip().startswith("|"):
j = i + 1
while j < len(lines) and lines[j].strip() == "":
j += 1
if j < len(lines) and lines[j].strip().startswith("|"):
i = j
continue
i += 1
return "\n".join(result)
_SEP_CELL = re.compile(r"^:?-+:?$")
def _is_sep_row(line: str) -> bool:
"""Return True when a pipe row is a Markdown table separator (| --- | --- |)."""
stripped = line.strip()
if not stripped.startswith("|"):
return False
cells = [c.strip() for c in stripped.split("|") if c.strip()]
return bool(cells) and all(_SEP_CELL.match(c) for c in cells)
def _merge_orphaned_table_rows(markdown: str) -> str:
"""Merge pipe-row blocks that lack a separator into the preceding table.
When the LLM incorrectly breaks a table (e.g. on a false group-header), it emits
orphaned pipe rows with no header or separator. These are invalid markdown and get
merged back into the preceding table, discarding the intervening non-table content.
"""
lines = markdown.split("\n")
segments: list[tuple[str, list[str]]] = []
i = 0
while i < len(lines):
if lines[i].strip().startswith("|"):
block: list[str] = []
while i < len(lines) and lines[i].strip().startswith("|"):
block.append(lines[i])
i += 1
has_sep = any(_is_sep_row(row) for row in block)
segments.append(("table" if has_sep else "orphan", block))
else:
block = []
while i < len(lines) and not lines[i].strip().startswith("|"):
block.append(lines[i])
i += 1
segments.append(("prose", block))
result: list[tuple[str, list[str]]] = []
last_table_idx: int | None = None
for seg_type, seg_lines in segments:
if seg_type == "orphan":
if last_table_idx is not None:
result = result[: last_table_idx + 1]
result[-1] = ("table", result[-1][1] + seg_lines)
else:
result.append((seg_type, seg_lines))
else:
if seg_type == "table":
last_table_idx = len(result)
result.append((seg_type, seg_lines))
return "\n".join(line for _, seg_lines in result for line in seg_lines)
def _remove_extra_separators(markdown: str) -> str:
"""Within each contiguous table block, keep only the first separator row."""
lines = markdown.split("\n")
result: list[str] = []
seen_sep = False
for line in lines:
if not line.strip().startswith("|"):
seen_sep = False
result.append(line)
continue
if _is_sep_row(line):
if seen_sep:
continue
seen_sep = True
result.append(line)
return "\n".join(result)
# ── Formatting helpers (module-level, no state) ──────────────────────────────────────────────────
def _format_layout(pages: list[PageLayout]) -> str:
if not pages:
return "None"
parts: list[str] = []
for page in pages:
line_strs: list[str] = []
for line in page.lines:
frags = " ".join(
f"{'**' if f.bold else ''}{f.text}{'**' if f.bold else ''}@({f.x:.0f},{f.y:.0f}) fs={f.font_size:.0f}"
for f in line.fragments
)
line_strs.append(f"y={line.y:.0f} | {frags}")
parts.append(f"--- Page {page.page_number} ---\n" + "\n".join(line_strs))
return "\n\n".join(parts)
@@ -1,10 +0,0 @@
"""Reasoning utilities shared across agents."""
from stirling.agents.shared.chunked_reasoner import ChunkedReasoner, ChunkNotes
from stirling.agents.shared.whole_doc_reader import WholeDocReaderCapability
__all__ = [
"ChunkNotes",
"ChunkedReasoner",
"WholeDocReaderCapability",
]
@@ -1,616 +0,0 @@
"""Chunked reasoning over long documents.
A reusable primitive for any agent that needs to answer a question that
requires reading a whole document end-to-end. The document is split into
character-budgeted chunks; each chunk is read by a parallel worker that
extracts question-relevant notes; if the gathered notes overflow the
synthesis context budget, the resulting notes are regrouped into fresh
chunks and run through the same extractor again, until they fit.
Pages are tracked by the wrapper, never asked of the model: keeps the model
output schema small and the page list authoritative.
Used wherever pure RAG retrieval is the wrong tool: aggregations ("largest
number"), comparisons ("shortest chapter"), and full summaries.
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass
from pydantic import BaseModel, Field
from pydantic_ai import Agent
from pydantic_ai.output import NativeOutput
from stirling.contracts import (
WholeDocCompressionRound,
WholeDocReadDone,
WholeDocReadStarted,
WholeDocSliceDone,
)
from stirling.contracts.documents import Page
from stirling.models import ApiModel
from stirling.services import AppRuntime, emit_progress
logger = logging.getLogger(__name__)
class ChunkNotes(ApiModel):
"""Public-facing notes for a span of pages.
Returned to callers of :meth:`ChunkedReasoner.gather_notes` and to the
inside of :meth:`ChunkedReasoner.reason`. The wrapper builds these from
the model's :class:`_ExtractedNotes` output and a deterministic page list.
"""
pages: list[int] = Field(description="Page numbers covered by these notes (1-indexed).")
summary: str = Field(description="One- to three-sentence summary of the covered range.")
relevant_excerpts: list[str] = Field(
default_factory=list,
description="Short verbatim quotes from the source content that bear on the user's question.",
)
facts: list[str] = Field(
default_factory=list,
description=(
"Concrete facts (numbers, names, dates, claims) the synthesiser may need. "
"Includes candidate values for aggregation questions."
),
)
class _ExtractedNotes(BaseModel):
"""Model output for one extractor call.
No ``pages`` field: page numbers are mechanical aggregation the wrapper
computes deterministically. Keeping them out of the schema saves output
tokens for the bulkier excerpts/facts payload and prevents the model
from misreporting page coverage.
"""
summary: str = Field(description="One- to three-sentence summary of the supplied content.")
relevant_excerpts: list[str] = Field(
default_factory=list,
description=(
"Short verbatim quotes drawn from the supplied content that bear on the question. "
"Deduplicate; drop ones that don't bear on the question."
),
)
facts: list[str] = Field(
default_factory=list,
description=(
"Distinct, deduplicated facts (numbers, names, dates, claims) needed to answer "
"the question. For aggregation questions retain ALL candidate values across the "
"supplied content so a later round can still pick the global winner."
),
)
@dataclass(frozen=True)
class _Chunk:
"""A unit of work for the extractor: content + the pages it covers + a fallback.
``content`` is the formatted text fed to the model: raw page text with
``[Page N]`` markers in the first round, formatted prior-pass notes with
``[Notes from pages A-B]`` markers in subsequent rounds. ``pages`` is
attached to the resulting :class:`ChunkNotes` deterministically.
``fallback`` is the list of notes to keep if the extractor call fails. For
raw page chunks it's empty (a failed slice has no pre-extracted notes to
preserve). For chunks built from existing notes it's the input notes
themselves, so a failure doesn't lose page coverage.
"""
content: str
pages: list[int]
fallback: list[ChunkNotes]
label: str
@dataclass(frozen=True)
class _RoundResult:
"""Outcome of one extraction round.
``successes`` lets the loop detect rounds that made no forward progress
(every chunk failed) and bail rather than spinning. ``slowest`` is the
chunk with the longest successful extractor call this round, used for
diagnostic log lines on the first round.
"""
notes: list[ChunkNotes]
successes: int
slowest: tuple[str, float] | None
def _page_range_label(pages: list[Page]) -> str:
if not pages:
return "pages=?"
elif len(pages) == 1:
return f"pages={pages[0].page_number}"
else:
return f"pages={pages[0].page_number}-{pages[-1].page_number}"
def _note_range_label(notes: list[ChunkNotes]) -> str:
"""Render a "pages=A-B" label for a group of already-extracted notes."""
page_numbers = sorted({p for note in notes for p in note.pages})
if not page_numbers:
return "pages=?"
if len(page_numbers) == 1:
return f"pages={page_numbers[0]}"
return f"pages={page_numbers[0]}-{page_numbers[-1]}"
_EXTRACTOR_SYSTEM_PROMPT = (
"You are reading content from a document - either raw page text or "
"condensed notes from an earlier extraction pass - and your job is to "
"produce a tight set of notes that captures everything relevant to the "
"user's question. The same job runs many times in parallel across the "
"document and may run again to consolidate notes into smaller batches, "
"so be thorough: anything you skip cannot be recovered later.\n"
"\n"
"Output:\n"
"- summary: 1-3 sentences covering the supplied content.\n"
"- relevant_excerpts: short verbatim quotes from the supplied content "
"that bear on the question. Deduplicate; drop quotes that don't help.\n"
"- facts: concrete facts (numbers, names, dates, claims). Deduplicate; "
"drop irrelevant ones. For aggregation questions (largest, smallest, "
"count, total) retain ALL candidate values across the content so a "
"later step can still pick the global winner.\n"
"\n"
"Stay grounded in the supplied content. Do not infer or fabricate "
"anything that isn't already present. If nothing in the content is "
"relevant to the question, return empty excerpts and facts and a short "
"neutral summary."
)
class ChunkedReasoner:
"""Run a question against a long document by chunking, mapping, and looping.
Two consumption styles:
* Tools that already have a synthesising LLM call upstream call
:meth:`gather_notes` to get the structured notes and format them
themselves with :meth:`format_notes`.
* Callers that just want an answer call :meth:`reason`, which runs
:meth:`gather_notes` and then a single synthesis call governed by the
caller's ``answer_prompt`` and ``answer_type``.
Lifetime:
Construct once per agent that uses it. The extractor agent is built
at construction time and reused; the synthesis agent in :meth:`reason`
is built per call because its output type is generic.
"""
def __init__(
self,
runtime: AppRuntime,
*,
chars_per_slice: int | None = None,
concurrency: int | None = None,
worker_timeout_seconds: float | None = None,
notes_char_budget: int | None = None,
) -> None:
chars = chars_per_slice if chars_per_slice is not None else runtime.settings.chunked_reasoner_chars_per_slice
conc = concurrency if concurrency is not None else runtime.settings.chunked_reasoner_concurrency
timeout = (
worker_timeout_seconds
if worker_timeout_seconds is not None
else runtime.settings.chunked_reasoner_worker_timeout_seconds
)
budget = (
notes_char_budget if notes_char_budget is not None else runtime.settings.chunked_reasoner_notes_char_budget
)
if chars <= 0:
raise ValueError("chars_per_slice must be positive")
if conc <= 0:
raise ValueError("concurrency must be positive")
if timeout <= 0:
raise ValueError("worker_timeout_seconds must be positive")
if budget <= 0:
raise ValueError("notes_char_budget must be positive")
self._runtime = runtime
self._chars_per_slice = chars
self._worker_timeout_seconds = timeout
self._notes_char_budget = budget
self._semaphore = asyncio.Semaphore(conc)
self._extractor: Agent[None, _ExtractedNotes] = Agent(
model=runtime.fast_model,
output_type=NativeOutput(_ExtractedNotes),
system_prompt=_EXTRACTOR_SYSTEM_PROMPT,
model_settings=runtime.fast_model_settings,
)
async def gather_notes(self, pages: list[Page], question: str) -> list[ChunkNotes]:
"""Return notes covering every page that fit the synthesis budget.
Worker failures are tolerated: surviving notes are returned. Returns
an empty list only when every first-round chunk raises, which the
caller can treat as a hard failure.
Progress events fire as each first-round chunk finishes (in completion
order, not chunk order) carrying a monotonic ``completed`` counter so
consumers can render "Read X of Y" with X advancing by exactly one
per event. Subsequent compression rounds emit a single round-start
event each.
"""
if not pages:
raise ValueError("ChunkedReasoner.gather_notes requires at least one page")
chunks = [self._chunk_from_pages(slice_pages) for slice_pages in self._slice_pages(pages)]
slice_total = len(chunks)
logger.info(
"[chunked-reasoner] question=%r pages=%d slices=%d",
question,
len(pages),
slice_total,
)
await emit_progress(WholeDocReadStarted(question=question, pages=len(pages), slices=slice_total))
gather_start = time.perf_counter()
notes = await self._run_chunks(chunks, question)
await emit_progress(
WholeDocReadDone(
completed=len(notes),
slices=slice_total,
duration_seconds=round(time.perf_counter() - gather_start, 2),
)
)
return notes
async def _run_chunks(self, chunks: list[_Chunk], question: str) -> list[ChunkNotes]:
"""Run chunks through the extractor, regrouping and looping until under budget.
The first round emits per-chunk progress events for streaming UIs;
later rounds emit a single round-start event. Each round may produce
fewer notes than chunks (every chunk maps to at most one consolidated
note); when the rendered notes still exceed the budget, the survivors
are regrouped into fresh chunks and the loop runs again.
"""
round_number = 0
while True:
chunks_in = len(chunks)
result = await self._extract_chunks(chunks, question, round_number)
if result.slowest is not None:
slow_label, slow_duration = result.slowest
logger.info(
"[chunked-reasoner] round %d: %d/%d chunks succeeded; slowest %s (%.1fs)",
round_number,
result.successes,
chunks_in,
slow_label,
slow_duration,
)
else:
logger.info(
"[chunked-reasoner] round %d: 0/%d chunks succeeded",
round_number,
chunks_in,
)
rendered_size = self._rendered_notes_size(result.notes)
if rendered_size <= self._notes_char_budget or len(result.notes) <= 1:
if round_number > 0:
logger.info(
"[chunked-reasoner] compression done after %d round(s): %d notes, %d chars",
round_number,
len(result.notes),
rendered_size,
)
return result.notes
if result.successes == 0:
# No forward progress this round; further rounds would
# reproduce the same shape. Return what we have.
logger.warning(
"[chunked-reasoner] round %d produced no successful extractions; bailing with %d notes",
round_number,
len(result.notes),
)
return result.notes
round_number += 1
groups = self._group_notes_for_compression(result.notes)
chunks = [self._chunk_from_notes(group) for group in groups]
logger.info(
"[chunked-reasoner] compression round %d: %d notes (%d chars) -> %d groups",
round_number,
len(result.notes),
rendered_size,
len(groups),
)
await emit_progress(
WholeDocCompressionRound(
round_number=round_number,
notes_in=len(result.notes),
groups=len(groups),
)
)
async def _extract_chunks(
self,
chunks: list[_Chunk],
question: str,
round_number: int,
) -> _RoundResult:
"""Run all chunks through the extractor in parallel; collect surviving notes.
Failures fall back to ``chunk.fallback`` (empty in the first round, so
failures drop; populated in compression rounds, so failures preserve
their input notes). The first round emits a
:class:`WholeDocSliceDone` per successful completion in completion
order, with a monotonic ``completed`` counter.
Returned notes are sorted by first page so downstream grouping packs
document-adjacent content together regardless of which task happened
to finish first.
"""
total = len(chunks)
pending: dict[asyncio.Task[tuple[ChunkNotes, float]], _Chunk] = {
asyncio.create_task(self._extract_chunk(chunk, question)): chunk for chunk in chunks
}
notes: list[ChunkNotes] = []
successes = 0
slowest: tuple[str, float] | None = None
completed = 0
try:
while pending:
done, _ = await asyncio.wait(pending.keys(), return_when=asyncio.FIRST_COMPLETED)
for task in done:
chunk = pending.pop(task)
exc = task.exception()
if exc is not None:
if chunk.fallback:
logger.warning(
"[chunked-reasoner] chunk %s failed: %s; preserving %d input note(s)",
chunk.label,
exc,
len(chunk.fallback),
)
notes.extend(chunk.fallback)
else:
logger.warning("[chunked-reasoner] chunk %s failed: %s", chunk.label, exc)
continue
extracted, duration = task.result()
notes.append(extracted)
successes += 1
completed += 1
if slowest is None or duration > slowest[1]:
slowest = (chunk.label, duration)
if round_number == 0:
await emit_progress(
WholeDocSliceDone(
completed=completed,
total=total,
pages=chunk.label,
duration_ms=int(duration * 1000),
excerpts=len(extracted.relevant_excerpts),
facts=len(extracted.facts),
)
)
finally:
# On cancellation (typically a frontend disconnect propagating up
# through the streaming orchestrator) the per-chunk model calls
# would otherwise keep running to completion, billing tokens whose
# results nobody is reading. Cancel and drain so the upstream
# cancellation is the cancellation that matters.
if pending:
for task in pending:
task.cancel()
await asyncio.gather(*pending.keys(), return_exceptions=True)
notes.sort(key=lambda n: n.pages[0] if n.pages else 0)
return _RoundResult(notes=notes, successes=successes, slowest=slowest)
async def _extract_chunk(self, chunk: _Chunk, question: str) -> tuple[ChunkNotes, float]:
"""Run the extractor on one chunk and attach the chunk's pages to the output."""
try:
extracted, duration = await self._run_extractor(chunk.content, question, chunk.label)
except TimeoutError:
logger.warning(
"[chunked-reasoner] chunk %s timed out (limit %.1fs)",
chunk.label,
self._worker_timeout_seconds,
)
raise
logger.debug(
"[chunked-reasoner] chunk %s: %d excerpt(s), %d fact(s) in %dms",
chunk.label,
len(extracted.relevant_excerpts),
len(extracted.facts),
int(duration * 1000),
)
return self._build_chunk_notes(extracted, chunk.pages), duration
async def _run_extractor(
self,
content: str,
question: str,
page_label: str,
) -> tuple[_ExtractedNotes, float]:
"""Inner primitive: run the extractor agent under semaphore + timeout."""
prompt = self._build_extraction_prompt(content, question)
async with self._semaphore:
start = time.perf_counter()
try:
result = await asyncio.wait_for(self._extractor.run(prompt), timeout=self._worker_timeout_seconds)
except TimeoutError:
duration = time.perf_counter() - start
logger.debug(
"[chunked-reasoner] extractor %s timed out after %dms",
page_label,
int(duration * 1000),
)
raise
duration = time.perf_counter() - start
return result.output, duration
def _chunk_from_pages(self, pages: list[Page]) -> _Chunk:
"""Build a first-round chunk from a slice of raw pages."""
return _Chunk(
content="\n\n".join(f"[Page {p.page_number}]\n{p.text}" for p in pages),
pages=[p.page_number for p in pages],
fallback=[],
label=_page_range_label(pages),
)
def _chunk_from_notes(self, group: list[ChunkNotes]) -> _Chunk:
"""Build a compression-round chunk from a group of prior-pass notes.
``fallback`` is the input group itself: if the extractor call fails,
the originals stay in the working set so page coverage isn't lost.
"""
return _Chunk(
content=self.format_notes(group),
pages=sorted({p for note in group for p in note.pages}),
fallback=group,
label=_note_range_label(group),
)
def _group_notes_for_compression(self, notes: list[ChunkNotes]) -> list[list[ChunkNotes]]:
"""Pack consecutive notes into groups whose rendered size fits ``chars_per_slice``.
Each group becomes one compression-round chunk. Sized to match the
first-round slice budget so the extractor sees roughly the same input
footprint regardless of which round is running. Single notes that
exceed the budget on their own become their own group.
"""
groups: list[list[ChunkNotes]] = []
current: list[ChunkNotes] = []
current_chars = 0
for note in notes:
note_chars = self._rendered_notes_size([note])
if current and current_chars + note_chars > self._chars_per_slice:
groups.append(current)
current = []
current_chars = 0
current.append(note)
current_chars += note_chars
if current:
groups.append(current)
return groups
@staticmethod
def _build_chunk_notes(extracted: _ExtractedNotes, pages: list[int]) -> ChunkNotes:
"""Build a public ChunkNotes from the model's output and the wrapper's pages."""
return ChunkNotes(
pages=pages,
summary=extracted.summary,
relevant_excerpts=extracted.relevant_excerpts,
facts=extracted.facts,
)
@staticmethod
def _build_extraction_prompt(content: str, question: str) -> str:
"""Single prompt shape used for every round.
The system prompt explains the role; the user prompt just hands over
the question and the content. Whether ``content`` is raw page text
with ``[Page N]`` markers or formatted notes with ``[Notes from
pages A-B]`` markers, the same instructions apply.
"""
return f"User question:\n{question}\n\nContent:\n{content}"
@staticmethod
def _rendered_notes_size(notes: list[ChunkNotes]) -> int:
"""Length in characters of what :meth:`format_notes` would produce."""
return len(ChunkedReasoner.format_notes(notes))
async def reason[T: BaseModel](
self,
*,
pages: list[Page],
question: str,
answer_prompt: str,
answer_type: type[T],
) -> T:
"""Map over pages, then synthesise a structured answer of type ``T``.
Args:
pages: Document pages in order.
question: The user's question, passed to both workers and the
synthesiser. Workers use it to decide what's relevant.
answer_prompt: System prompt for the synthesis stage. Should
instruct the model to answer ``question`` from the notes
supplied. Owned by the caller because the answer's tone,
format, and grounding rules are domain-specific.
answer_type: Pydantic model describing the structured answer.
Returns:
An instance of ``answer_type`` produced by the synthesis stage.
"""
notes = await self.gather_notes(pages, question)
if not notes:
raise RuntimeError("All chunked-reasoning workers failed; no notes to synthesise from")
return await self._synthesise(question, notes, answer_prompt, answer_type)
@staticmethod
def format_notes(notes: list[ChunkNotes]) -> str:
"""Render notes as readable text for inclusion in another agent's tool result.
Order is preserved. Page numbers, summary, excerpts and facts are all
emitted; empty sections are omitted.
"""
sections: list[str] = []
for n in notes:
page_label = (
f"pages {n.pages[0]}-{n.pages[-1]}"
if len(n.pages) > 1
else f"page {n.pages[0]}"
if n.pages
else "unknown pages"
)
block = [f"[Notes from {page_label}]", f"Summary: {n.summary}"]
if n.relevant_excerpts:
block.append("Relevant excerpts:")
block.extend(f"- {e}" for e in n.relevant_excerpts)
if n.facts:
block.append("Facts:")
block.extend(f"- {f}" for f in n.facts)
sections.append("\n".join(block))
return "\n\n".join(sections)
def _slice_pages(self, pages: list[Page]) -> list[list[Page]]:
"""Group consecutive pages into character-budgeted slices.
Page boundaries are preserved: a single page is never split across
slices. If one page exceeds the budget on its own, it becomes its
own slice.
"""
slices: list[list[Page]] = []
current: list[Page] = []
current_chars = 0
for page in pages:
if current and current_chars + page.char_count > self._chars_per_slice:
slices.append(current)
current = []
current_chars = 0
current.append(page)
current_chars += page.char_count
if current:
slices.append(current)
return slices
async def _synthesise[T: BaseModel](
self,
question: str,
notes: list[ChunkNotes],
answer_prompt: str,
answer_type: type[T],
) -> T:
agent: Agent[None, T] = Agent(
model=self._runtime.smart_model,
output_type=NativeOutput(answer_type),
system_prompt=answer_prompt,
model_settings=self._runtime.smart_model_settings,
)
prompt = f"User question:\n{question}\n\nNotes from across the document:\n\n{self.format_notes(notes)}"
result = await agent.run(prompt)
return result.output
@@ -1,140 +0,0 @@
"""Tool capability that lets an agent read whole documents end-to-end.
Companion to :class:`stirling.documents.RagCapability`. Where ``RagCapability``
gives an agent targeted vector retrieval, this gives it map-style whole-document
reading: every page is read in parallel by fast-model workers, and the
question-relevant notes are returned for the agent to synthesise.
Use both capabilities together when the agent should pick its strategy:
``search_knowledge`` for specific lookups, ``read_full_document`` for
aggregations, comparisons, and summaries.
"""
from __future__ import annotations
import logging
from pydantic_ai import FunctionToolset, RunContext, ToolDefinition
from pydantic_ai.toolsets import AbstractToolset
from stirling.agents.shared.chunked_reasoner import ChunkedReasoner
from stirling.contracts import AiFile
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
# Cap on per-run calls. One pass already reads every page of every attached
# document, so a second call is almost always the smart model second-guessing
# itself on a near-identical query (and doubles wall-clock time for a sizeable
# document). If a follow-up genuinely needs more, ``search_knowledge`` is the
# right escape hatch. Configurable per-construction in case a future caller
# can prove a real two-read use case; the default stays at 1.
DEFAULT_MAX_READS = 1
class WholeDocReaderCapability:
"""Bundles instructions and the ``read_full_document`` toolset for agent injection.
Lifecycle: a ``WholeDocReaderCapability`` instance is intended to live for
the duration of a single agent run.
The agent picks between this and :class:`RagCapability` per the tool
descriptions: targeted retrieval vs whole-document reading.
"""
def __init__(
self,
runtime: AppRuntime,
files: list[AiFile],
*,
reasoner: ChunkedReasoner | None = None,
max_reads: int = DEFAULT_MAX_READS,
) -> None:
self._runtime = runtime
self._files = files
self._reasoner = reasoner if reasoner is not None else ChunkedReasoner(runtime)
self._max_reads = max_reads
self._read_count = 0
toolset: FunctionToolset[None] = FunctionToolset()
toolset.add_function(
self._read_full_document,
name="read_full_document",
prepare=self._prepare_read_full_document,
)
self._toolset = toolset
@property
def instructions(self) -> str:
names = ", ".join(f.name for f in self._files) if self._files else "the attached documents"
return (
"You have a 'read_full_document' tool that reads every page of "
f"{names} in parallel and returns notes relevant to a query. "
"Use it when answering requires seeing the whole document end-to-end "
"(summaries, aggregations, comparisons across sections). One call "
"already reads everything; phrase the query to cover all the angles "
"you need in a single pass. For follow-ups or specific lookups use "
"'search_knowledge', which is cheaper and targeted."
)
@property
def toolset(self) -> AbstractToolset[None]:
return self._toolset
async def _prepare_read_full_document(
self,
ctx: RunContext[None],
tool_def: ToolDefinition,
) -> ToolDefinition | None:
"""Hide the tool from the agent's toolset once the per-run budget is spent.
Mirrors the search_knowledge prepare callback."""
if self._read_count >= self._max_reads:
return None
return tool_def
async def _read_full_document(self, query: str) -> str:
"""Read every page of the attached documents and return notes relevant to the query.
Use this when answering needs the whole document end-to-end - summaries,
aggregations like 'largest number' or 'shortest chapter', or comparisons
across sections. Slow and expensive (one fast-model call per slice per
document); prefer search_knowledge for targeted lookups.
Args:
query: A focused description of what to extract from the documents,
phrased so a worker reading just one slice can decide what's
relevant to the user's question.
Returns:
Per-document sections of structured notes (page numbers, summary,
relevant excerpts, extracted facts), already ordered by page.
"""
self._read_count += 1
if not self._files:
return "No documents attached to read."
sections: list[str] = []
for file in self._files:
pages = await self._runtime.documents.read_pages(file.id)
if not pages:
logger.info(
"[whole-doc-reader] no stored pages for %s (id=%s); skipping",
file.name,
file.id,
)
continue
notes = await self._reasoner.gather_notes(pages, query)
if not notes:
continue
sections.append(f"=== {file.name} ===\n{ChunkedReasoner.format_notes(notes)}")
if not sections:
return "Could not read any document content."
logger.info(
"[whole-doc-reader] read query=%r files=%d -> %d chars",
query,
len(self._files),
sum(len(s) for s in sections),
)
return "\n\n".join(sections)
+3 -3
View File
@@ -19,13 +19,13 @@ from stirling.agents.pdf_comment import PdfCommentAgent
from stirling.api.middleware import UserIdMiddleware
from stirling.api.routes import (
agent_draft_router,
document_router,
execution_router,
ledger_router,
orchestrator_router,
pdf_comments_router,
pdf_edit_router,
pdf_question_router,
rag_router,
)
from stirling.config import AppSettings, load_settings
from stirling.contracts import HealthResponse
@@ -57,7 +57,7 @@ async def lifespan(fast_api: FastAPI):
if tracer_provider:
Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider))
yield
await runtime.documents.close()
await runtime.rag_service.close()
if tracer_provider:
tracer_provider.shutdown()
@@ -69,7 +69,7 @@ app.include_router(pdf_edit_router)
app.include_router(pdf_question_router)
app.include_router(agent_draft_router)
app.include_router(execution_router)
app.include_router(document_router)
app.include_router(rag_router)
app.include_router(ledger_router)
app.include_router(pdf_comments_router)
+3 -3
View File
@@ -11,7 +11,7 @@ from stirling.agents import (
)
from stirling.agents.ledger import MathAuditorAgent
from stirling.agents.pdf_comment import PdfCommentAgent
from stirling.documents import DocumentService
from stirling.rag import RagService
from stirling.services import AppRuntime
@@ -39,8 +39,8 @@ def get_execution_planning_agent(request: Request) -> ExecutionPlanningAgent:
return request.app.state.execution_planning_agent
def get_document_service(request: Request) -> DocumentService:
return request.app.state.runtime.documents
def get_rag_service(request: Request) -> RagService:
return request.app.state.runtime.rag_service
def get_math_auditor_agent(request: Request) -> MathAuditorAgent:
+2 -2
View File
@@ -1,19 +1,19 @@
from .agent_drafts import router as agent_draft_router
from .documents import router as document_router
from .execution import router as execution_router
from .ledger import router as ledger_router
from .orchestrator import router as orchestrator_router
from .pdf_comments import router as pdf_comments_router
from .pdf_edit import router as pdf_edit_router
from .pdf_questions import router as pdf_question_router
from .rag import router as rag_router
__all__ = [
"agent_draft_router",
"document_router",
"execution_router",
"ledger_router",
"orchestrator_router",
"pdf_comments_router",
"pdf_edit_router",
"pdf_question_router",
"rag_router",
]
@@ -1,49 +0,0 @@
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, Depends
from stirling.api.dependencies import get_document_service
from stirling.contracts import (
DeleteDocumentResponse,
IngestDocumentRequest,
IngestDocumentResponse,
)
from stirling.documents import DocumentService
from stirling.models import FileId
router = APIRouter(prefix="/api/v1/documents", tags=["documents"])
@router.post("", response_model=IngestDocumentResponse)
async def ingest_document(
request: IngestDocumentRequest,
documents: Annotated[DocumentService, Depends(get_document_service)],
) -> IngestDocumentResponse:
"""Replace-ingest a document's content under ``document_id``.
Stores both representations in one shot:
* embedded chunks for RAG search,
* ordered page text for whole-document reading.
Any previously-stored content for this document is removed first.
"""
pages = request.page_text or []
chunks_indexed = await documents.ingest(
collection=request.document_id,
pages=pages,
source=request.source,
)
return IngestDocumentResponse(document_id=request.document_id, chunks_indexed=chunks_indexed)
@router.delete("/{document_id}", response_model=DeleteDocumentResponse)
async def delete_document(
document_id: FileId,
documents: Annotated[DocumentService, Depends(get_document_service)],
) -> DeleteDocumentResponse:
"""Remove a document's content. Idempotent."""
existed = await documents.has_collection(document_id)
if existed:
await documents.delete_collection(document_id)
return DeleteDocumentResponse(document_id=document_id, deleted=existed)
+5 -156
View File
@@ -1,170 +1,19 @@
from __future__ import annotations
import asyncio
import json
import logging
from collections.abc import AsyncIterator
from dataclasses import dataclass
from typing import Annotated, assert_never
from typing import Annotated
from fastapi import APIRouter, Depends
from fastapi.responses import StreamingResponse
from stirling.agents import OrchestratorAgent
from stirling.api.dependencies import get_orchestrator_agent
from stirling.contracts import OrchestratorRequest, OrchestratorResponse, ProgressEvent
from stirling.services import reset_progress_emitter, set_progress_emitter
logger = logging.getLogger(__name__)
# Cadence for keep-alive heartbeats on the streaming endpoint. Java forwards
# them to the frontend as SSE comments; their job is to make every layer of
# the connection visibly alive at this rhythm so disconnects surface within a
# bounded window instead of waiting for the next progress event.
HEARTBEAT_INTERVAL_SECONDS = 10.0
from stirling.contracts import OrchestratorRequest, OrchestratorResponse
router = APIRouter(prefix="/api/v1/orchestrator", tags=["orchestrator"])
@router.post("")
@router.post("", response_model=OrchestratorResponse)
async def orchestrate(
request: OrchestratorRequest,
agent: Annotated[OrchestratorAgent, Depends(get_orchestrator_agent)],
) -> StreamingResponse:
"""Run the orchestrator and stream NDJSON events.
Each output line is a JSON object with an ``event`` field. ``progress``
events arrive whenever an inner agent reports work (e.g. each
chunked-reasoner slice completing); the final ``result`` event carries the
typed orchestrator response. ``error`` events surface failures without
breaking the connection. ``heartbeat`` events fire on a fixed cadence to
keep idle connections visibly alive so disconnects propagate.
The stream itself is the liveness signal: as long as events flow, work is
alive. Java consumes this with a long total timeout and treats line
arrival as forward progress.
"""
return StreamingResponse(
_OrchestratorStream(
agent=agent,
request=request,
heartbeat_interval_seconds=HEARTBEAT_INTERVAL_SECONDS,
).iterate(),
media_type="application/x-ndjson",
)
@dataclass(frozen=True, slots=True)
class _ProgressFrame:
event: ProgressEvent
@dataclass(frozen=True, slots=True)
class _ResultFrame:
response: OrchestratorResponse
@dataclass(frozen=True, slots=True)
class _ErrorFrame:
message: str
@dataclass(frozen=True, slots=True)
class _HeartbeatFrame:
"""No payload: a heartbeat exists only to push bytes through the pipe.
Without periodic traffic, a slow workflow phase (e.g. all extractor
workers busy on long calls) leaves the engine writer, Java's SSE
forwarder, and the frontend's fetch all silently waiting. A closed
connection at any layer wouldn't surface until the next real event,
which could be many tens of seconds away. Heartbeats bound that window
to :data:`HEARTBEAT_INTERVAL_SECONDS`.
"""
type _StreamFrame = _ProgressFrame | _ResultFrame | _ErrorFrame | _HeartbeatFrame
def _serialize_frame(frame: _StreamFrame) -> bytes:
"""Render a frame as one NDJSON line."""
match frame:
case _ProgressFrame(event=event):
body = {"event": "progress", **event.model_dump(mode="json")}
case _ResultFrame(response=response):
body = {"event": "result", "response": response.model_dump(mode="json")}
case _ErrorFrame(message=message):
body = {"event": "error", "message": message}
case _HeartbeatFrame():
body = {"event": "heartbeat"}
case _:
assert_never(frame)
return (json.dumps(body) + "\n").encode("utf-8")
class _OrchestratorStream:
"""Drives one streaming orchestrator request.
Owns the per-request queue and pumps progress events through it; the agent
runs as a child task so its emissions and the streaming response interleave.
A heartbeat task pushes keep-alive messages onto the same queue at a fixed
cadence so the connection stays visibly alive between progress events.
"""
def __init__(
self,
*,
agent: OrchestratorAgent,
request: OrchestratorRequest,
heartbeat_interval_seconds: float,
) -> None:
self._agent = agent
self._request = request
self._heartbeat_interval_seconds = heartbeat_interval_seconds
self._queue: asyncio.Queue[_StreamFrame | None] = asyncio.Queue()
async def iterate(self) -> AsyncIterator[bytes]:
token = set_progress_emitter(self._emit_progress)
agent_task = asyncio.create_task(self._run_agent())
heartbeat_task = asyncio.create_task(self._emit_heartbeats())
try:
while True:
frame = await self._queue.get()
if frame is None:
break
yield _serialize_frame(frame)
finally:
reset_progress_emitter(token)
await self._cancel_task(heartbeat_task)
await self._cancel_task(agent_task)
async def _emit_progress(self, event: ProgressEvent) -> None:
await self._queue.put(_ProgressFrame(event=event))
async def _emit_heartbeats(self) -> None:
while True:
await asyncio.sleep(self._heartbeat_interval_seconds)
await self._queue.put(_HeartbeatFrame())
async def _run_agent(self) -> None:
try:
response = await self._agent.handle(self._request)
await self._queue.put(_ResultFrame(response=response))
except asyncio.CancelledError:
raise
except Exception as exc:
logger.exception("orchestrator stream failed")
await self._queue.put(_ErrorFrame(message=str(exc)))
finally:
await self._queue.put(None)
@staticmethod
async def _cancel_task(task: asyncio.Task[None]) -> None:
if task.done():
return
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("background task failed during cancellation", exc_info=True)
) -> OrchestratorResponse:
return await agent.handle(request)
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, Depends
from stirling.api.dependencies import get_rag_service
from stirling.contracts import (
DeleteDocumentResponse,
IngestDocumentRequest,
IngestDocumentResponse,
PdfContentType,
)
from stirling.models import FileId
from stirling.rag import Document, RagService
router = APIRouter(prefix="/api/v1/rag", tags=["rag"])
@router.post("/documents", response_model=IngestDocumentResponse)
async def ingest_document(
request: IngestDocumentRequest,
rag: Annotated[RagService, Depends(get_rag_service)],
) -> IngestDocumentResponse:
"""Replace-ingest a document's content under ``document_id``.
Any previously-stored content for this document is removed and the
provided content replaces it wholesale. All pages are chunked up front
and then embedded in a single batched call so large documents (e.g. a
500-page book) don't fan out into hundreds of embedding requests.
"""
await rag.delete_collection(request.document_id)
chunks: list[Document] = []
if request.page_text:
for page in request.page_text:
if not page.text.strip():
continue
chunks.extend(
rag.chunk_text(
text=page.text,
source=f"{request.source}:page:{page.page_number}",
base_metadata={
"page_number": str(page.page_number),
"content_type": PdfContentType.PAGE_TEXT.value,
},
)
)
indexed = await rag.index_documents(request.document_id, chunks) if chunks else 0
return IngestDocumentResponse(document_id=request.document_id, chunks_indexed=indexed)
@router.delete("/documents/{document_id}", response_model=DeleteDocumentResponse)
async def delete_document(
document_id: FileId,
rag: Annotated[RagService, Depends(get_rag_service)],
) -> DeleteDocumentResponse:
"""Remove a document's content from RAG. Idempotent."""
existed = await rag.has_collection(document_id)
if existed:
await rag.delete_collection(document_id)
return DeleteDocumentResponse(document_id=document_id, deleted=existed)
+2 -54
View File
@@ -38,38 +38,18 @@ class AppSettings(BaseSettings):
rag_default_top_k: int = Field(validation_alias="STIRLING_RAG_TOP_K")
rag_max_searches: int = Field(validation_alias="STIRLING_RAG_MAX_SEARCHES")
# Chunked reasoner settings (whole-document map-reduce).
chunked_reasoner_chars_per_slice: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_CHARS_PER_SLICE")
chunked_reasoner_concurrency: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_CONCURRENCY")
chunked_reasoner_worker_timeout_seconds: float = Field(
validation_alias="STIRLING_CHUNKED_REASONER_WORKER_TIMEOUT_SECONDS"
)
# Maximum size, in characters, of the rendered notes block before the
# reasoner folds slice notes hierarchically. The Anthropic context limit
# is 200k tokens (~880k chars); we leave a generous margin for the
# downstream agent's system prompt, history, tool definitions, and
# response budget.
chunked_reasoner_notes_char_budget: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_NOTES_CHAR_BUDGET")
max_pages: int = Field(validation_alias="STIRLING_MAX_PAGES")
max_characters: int = Field(validation_alias="STIRLING_MAX_CHARACTERS")
log_level: str = Field(default="INFO", validation_alias="STIRLING_LOG_LEVEL")
log_file: str = Field(default="", validation_alias="STIRLING_LOG_FILE")
# When true, raises httpx + httpcore logger levels so every outgoing
# model SDK call is logged with timing. Use to diagnose worker stalls:
# a hung request shows the "Request: POST ..." line with no matching
# response line, confirming the hang is transport-layer (not in our
# code or the Anthropic SDK itself). Off by default — DEBUG-level
# output is high-volume.
http_debug: bool = Field(default=False, validation_alias="STIRLING_HTTP_DEBUG")
posthog_enabled: bool = Field(validation_alias="STIRLING_POSTHOG_ENABLED")
posthog_api_key: str = Field(validation_alias="STIRLING_POSTHOG_API_KEY")
posthog_host: str = Field(validation_alias="STIRLING_POSTHOG_HOST")
def _configure_logging(level_name: str, log_file: str, http_debug: bool) -> None:
def _configure_logging(level_name: str, log_file: str) -> None:
"""Configure the ``stirling`` logger hierarchy."""
level = logging.getLevelNamesMapping().get(level_name.upper())
if level is None:
@@ -103,43 +83,11 @@ def _configure_logging(level_name: str, log_file: str, http_debug: bool) -> None
fh.setLevel(level)
root.addHandler(fh)
if http_debug:
_enable_http_debug(formatter)
def _enable_http_debug(formatter: logging.Formatter) -> None:
"""Surface every httpx/httpcore call against the Anthropic API.
httpx emits one INFO line per request with the URL and final status,
which is the most useful signal for diagnosing hung worker calls: a
successful call shows "Request" then "Response" within a second or two;
a hung one shows "Request" with no matching response until it's
cancelled. httpcore at DEBUG drills down to TCP / HTTP/2 stream events
if the user wants to see exactly where bytes stop flowing.
The ``stirling`` console handler is scoped to its own logger tree, so
we attach a dedicated stream handler here. Without it, httpx records
propagate to the root logger which has no handler in our setup and the
output is silently dropped.
"""
handler = logging.StreamHandler()
handler.setFormatter(formatter)
handler.setLevel(logging.DEBUG)
for name, level in (("httpx", logging.INFO), ("httpcore", logging.DEBUG)):
lg = logging.getLogger(name)
lg.setLevel(level)
# Idempotent: avoid stacking handlers on settings reload.
if not any(getattr(h, "_stirling_http_debug", False) for h in lg.handlers):
handler._stirling_http_debug = True # type: ignore[attr-defined]
lg.addHandler(handler)
lg.propagate = False
@lru_cache(maxsize=1)
def load_settings() -> AppSettings:
load_dotenv(ENV_FILE)
load_dotenv(ENV_LOCAL_FILE, override=True)
settings = AppSettings.model_validate({})
_configure_logging(settings.log_level, settings.log_file, settings.http_debug)
_configure_logging(settings.log_level, settings.log_file)
return settings
+6 -44
View File
@@ -14,7 +14,6 @@ from .common import (
ArtifactKind,
ConversationMessage,
ExtractedFileText,
GenerateFileResponse,
MathAuditorToolReportArtifact,
NeedContentFileRequest,
NeedContentResponse,
@@ -29,14 +28,6 @@ from .common import (
format_conversation_history,
format_file_names,
)
from .documents import (
DeleteDocumentResponse,
IngestDocumentRequest,
IngestDocumentResponse,
Page,
PageRange,
PageText,
)
from .execution import (
AgentExecutionRequest,
CannotContinueExecutionAction,
@@ -88,23 +79,11 @@ from .pdf_questions import (
PdfQuestionResponse,
PdfQuestionTerminalResponse,
)
from .pdf_to_markdown import (
LayoutFragment,
LayoutLine,
PageLayout,
PageLayoutArtifact,
PageLayoutFileEntry,
PdfToMarkdownCannotDoResponse,
PdfToMarkdownOrchestrateResponse,
PdfToMarkdownRequest,
PdfToMarkdownResponse,
)
from .progress import (
ProgressEvent,
WholeDocCompressionRound,
WholeDocReadDone,
WholeDocReadStarted,
WholeDocSliceDone,
from .rag import (
DeleteDocumentResponse,
IngestDocumentRequest,
IngestDocumentResponse,
IngestedPageText,
)
__all__ = [
@@ -126,10 +105,6 @@ __all__ = [
"CompletedExecutionAction",
"ConversationMessage",
"DeleteDocumentResponse",
"PdfToMarkdownCannotDoResponse",
"PdfToMarkdownOrchestrateResponse",
"PdfToMarkdownRequest",
"PdfToMarkdownResponse",
"Discrepancy",
"DiscrepancyKind",
"EditCannotDoResponse",
@@ -145,10 +120,10 @@ __all__ = [
"FolioType",
"format_conversation_history",
"format_file_names",
"GenerateFileResponse",
"HealthResponse",
"IngestDocumentRequest",
"IngestDocumentResponse",
"IngestedPageText",
"MathAuditorToolReportArtifact",
"NeedContentFileRequest",
"NeedContentResponse",
@@ -156,14 +131,6 @@ __all__ = [
"NextExecutionAction",
"OrchestratorRequest",
"OrchestratorResponse",
"LayoutFragment",
"LayoutLine",
"Page",
"PageLayout",
"PageLayoutArtifact",
"PageLayoutFileEntry",
"PageRange",
"PageText",
"PdfCommentInstruction",
"PdfCommentReport",
"PdfCommentRequest",
@@ -179,7 +146,6 @@ __all__ = [
"PdfQuestionResponse",
"PdfQuestionTerminalResponse",
"PdfTextSelection",
"ProgressEvent",
"Requisition",
"Severity",
"StepKind",
@@ -190,10 +156,6 @@ __all__ = [
"ToolReportArtifact",
"UnsupportedCapabilityResponse",
"Verdict",
"WholeDocCompressionRound",
"WholeDocReadDone",
"WholeDocReadStarted",
"WholeDocSliceDone",
"WorkflowArtifact",
"WorkflowOutcome",
]
+2 -18
View File
@@ -61,7 +61,6 @@ class WorkflowOutcome(StrEnum):
COMPLETED = "completed"
CANNOT_CONTINUE = "cannot_continue"
UNSUPPORTED_CAPABILITY = "unsupported_capability"
GENERATE_FILE = "generate_file"
class ArtifactKind(StrEnum):
@@ -71,7 +70,6 @@ class ArtifactKind(StrEnum):
"""
EXTRACTED_TEXT = "extracted_text"
PAGE_LAYOUT = "page_layout"
TOOL_REPORT = "tool_report"
@@ -91,7 +89,6 @@ class SupportedCapability(StrEnum):
AGENT_REVISE = "agent_revise"
AGENT_NEXT_ACTION = "agent_next_action"
MATH_AUDITOR_AGENT = "math_auditor_agent"
PDF_TO_MARKDOWN = "pdf_to_markdown"
class ConversationMessage(ApiModel):
@@ -170,10 +167,10 @@ ToolReportArtifact = MathAuditorToolReportArtifact
class NeedIngestResponse(ApiModel):
"""Signal that the listed files must be ingested before the agent can continue.
"""Signal that the listed files must be ingested into RAG before the agent can continue.
Java's handling: for each file, extract the requested content types, POST to
``/api/v1/documents`` keyed by ``file.id``, then retry the original request.
``/api/v1/rag/documents`` keyed by ``file.id``, then retry the original request.
"""
outcome: Literal[WorkflowOutcome.NEED_INGEST] = WorkflowOutcome.NEED_INGEST
@@ -203,19 +200,6 @@ class ToolOperationStep(ApiModel):
return self
class GenerateFileResponse(ApiModel):
"""Return generated text content directly to Java for file packaging.
Java converts the content string to bytes and stores it as a result file,
avoiding a round-trip through a write-file tool endpoint.
"""
outcome: Literal[WorkflowOutcome.GENERATE_FILE] = WorkflowOutcome.GENERATE_FILE
content: str
filename: str = Field(pattern=r"^[^/\\]+$", description="Output filename; no path separators.")
summary: str | None = None
def drop_unknown_tool_endpoints(value: Iterable[str | ToolEndpoint]) -> list[ToolEndpoint]:
"""Coerce inbound endpoint identifiers into `ToolEndpoint` members, dropping unknowns.
@@ -1,61 +0,0 @@
from __future__ import annotations
from pydantic import Field
from stirling.models import ApiModel
from .common import FileId
class PageText(ApiModel):
"""A single page of extracted text on the ingest wire."""
page_number: int = Field(ge=1)
text: str
class Page(ApiModel):
"""A single page of a document, retrieved from storage.
``char_count`` is precomputed at ingest time and reported here so callers
can budget how much content they want to read without first concatenating
the text of every page.
"""
page_number: int = Field(ge=1)
text: str
char_count: int = Field(ge=0)
class PageRange(ApiModel):
"""Inclusive page range for partial reads. Both bounds are 1-indexed."""
start: int = Field(ge=1)
end: int = Field(ge=1)
class IngestDocumentRequest(ApiModel):
"""Replace-ingest a document's content under the given ``document_id``.
Each call wipes any previously-stored content for the document and writes
both the vector-chunk and ordered-page representations from the supplied
pages.
``source`` is a human-readable label (typically the original filename)
that flows into chunk metadata so search results are readable when
``document_id`` is a hash.
"""
document_id: FileId = Field(min_length=1)
source: str = Field(min_length=1)
page_text: list[PageText] | None = None
class IngestDocumentResponse(ApiModel):
document_id: FileId
chunks_indexed: int
class DeleteDocumentResponse(ApiModel):
document_id: FileId
deleted: bool
@@ -12,7 +12,6 @@ from .common import (
ArtifactKind,
ConversationMessage,
ExtractedFileText,
GenerateFileResponse,
NeedContentResponse,
NeedIngestResponse,
SupportedCapability,
@@ -23,7 +22,6 @@ from .common import (
from .execution import NextExecutionAction
from .pdf_edit import PdfEditTerminalResponse
from .pdf_questions import PdfQuestionTerminalResponse
from .pdf_to_markdown import PageLayoutArtifact
class ExtractedTextArtifact(ApiModel):
@@ -31,10 +29,7 @@ class ExtractedTextArtifact(ApiModel):
files: list[ExtractedFileText] = Field(default_factory=list)
WorkflowArtifact = Annotated[
ExtractedTextArtifact | PageLayoutArtifact | ToolReportArtifact,
Field(discriminator="kind"),
]
WorkflowArtifact = Annotated[ExtractedTextArtifact | ToolReportArtifact, Field(discriminator="kind")]
class OrchestratorRequest(ApiModel):
@@ -58,7 +53,6 @@ class UnsupportedCapabilityResponse(ApiModel):
type OrchestratorResponse = Annotated[
PdfEditTerminalResponse
| PdfQuestionTerminalResponse
| GenerateFileResponse
| NeedContentResponse
| NeedIngestResponse
| AgentDraftResponse
@@ -1,105 +0,0 @@
"""Contracts for the PDF to Markdown Agent.
The agent accepts a parsed document and returns a single Markdown document that
faithfully reconstructs the PDF content — headings, paragraphs, and tables in
reading order, using page_layout as the primary source of truth for structure.
Java extracts page layout via PdfIngester and returns it as a PageLayoutArtifact
through the orchestrator resume_with pattern.
"""
from __future__ import annotations
from typing import Annotated, Literal
from pydantic import Field
from stirling.models import ApiModel
from .common import ArtifactKind, ConversationMessage, GenerateFileResponse, NeedContentResponse
from .pdf_edit import EditCannotDoResponse
# ── Input: layout models (mirror Java's RawLine / TextFragment geometry) ────────────────────────
class LayoutFragment(ApiModel):
"""One text fragment with its bounding-box geometry and font properties."""
text: str
x: float
y: float
width: float
font_size: float
bold: bool
class LayoutLine(ApiModel):
"""A visual line on the page: one y-coordinate and all fragments on that line."""
y: float
fragments: list[LayoutFragment]
class PageLayout(ApiModel):
"""All layout lines for a single page, in top-to-bottom order."""
page_number: int
lines: list[LayoutLine]
# ── Artifact: page layout (produced by Java, consumed by orchestrate()) ──────────────────────────
class PageLayoutFileEntry(ApiModel):
"""Page layout data for one file, as extracted by Java's PdfIngester."""
file_name: str
pages: list[PageLayout] = Field(default_factory=list)
class PageLayoutArtifact(ApiModel):
"""Artifact carrying full spatial page layout for all input files."""
kind: Literal[ArtifactKind.PAGE_LAYOUT] = ArtifactKind.PAGE_LAYOUT
files: list[PageLayoutFileEntry] = Field(default_factory=list)
# ── Input: full request ──────────────────────────────────────────────────────────────────────────
class PdfToMarkdownRequest(ApiModel):
"""Request sent by Java after PdfIngester has parsed the document.
page_layout: per-fragment positional data from the original (y-sorted) line order.
Each fragment carries its x/y position, width, font size, and bold flag.
This is the primary source of truth for column detection and heading hierarchy.
"""
user_message: str
file_names: list[str] = Field(default_factory=list)
conversation_history: list[ConversationMessage] = Field(default_factory=list)
page_layout: list[PageLayout] = Field(default_factory=list)
# ── Output: response variants ────────────────────────────────────────────────────────────────────
class PdfToMarkdownSuccessResponse(ApiModel):
outcome: Literal["document_reconstructed"] = "document_reconstructed"
markdown: str
class PdfToMarkdownCannotDoResponse(ApiModel):
outcome: Literal["cannot_do"] = "cannot_do"
reason: str
type PdfToMarkdownResponse = Annotated[
PdfToMarkdownSuccessResponse | PdfToMarkdownCannotDoResponse,
Field(discriminator="outcome"),
]
type PdfToMarkdownOrchestrateResponse = Annotated[
GenerateFileResponse | EditCannotDoResponse | NeedContentResponse,
Field(discriminator="outcome"),
]
-70
View File
@@ -1,70 +0,0 @@
"""Progress events emitted by deep callees during a streaming orchestrator run.
Each subclass models one engine-side phase. The Java side forwards the JSON
verbatim into ``AiWorkflowProgressEvent.engineDetail``; the frontend switches
on ``phase`` and renders the typed fields.
"""
from __future__ import annotations
from typing import Annotated, Literal
from pydantic import Field
from stirling.models import ApiModel
class WholeDocReadStarted(ApiModel):
phase: Literal["whole_doc_read_started"] = "whole_doc_read_started"
question: str
pages: int
slices: int
class WholeDocSliceDone(ApiModel):
"""Emitted as each chunked-reasoner worker completes.
``completed`` is a monotonically increasing counter (1..total) reflecting
the order in which workers finished, NOT the slice's position in the
document. Callers showing "Read X of Y" should use this directly so X
increments by one with each event.
"""
phase: Literal["whole_doc_slice_done"] = "whole_doc_slice_done"
completed: int
total: int
pages: str
duration_ms: int
excerpts: int
facts: int
class WholeDocCompressionRound(ApiModel):
"""Emitted when the gathered slice notes exceed the synthesis context
budget and the reasoner consolidates them with a fast-model fold pass.
Long documents (a 3000-page novel produces ~900k chars of raw notes)
would otherwise overflow the smart-model's prompt. ``notes_in`` is the
count entering the round; ``groups`` is the number of fold calls fired
(each producing one consolidated note). One or two rounds usually fit;
the event fires per round so callers can render "Consolidating notes
(round N)..." rather than going silent through the fold.
"""
phase: Literal["whole_doc_compression_round"] = "whole_doc_compression_round"
round_number: int
notes_in: int
groups: int
class WholeDocReadDone(ApiModel):
phase: Literal["whole_doc_read_done"] = "whole_doc_read_done"
completed: int
slices: int
duration_seconds: float
type ProgressEvent = Annotated[
WholeDocReadStarted | WholeDocSliceDone | WholeDocCompressionRound | WholeDocReadDone,
Field(discriminator="phase"),
]
+39
View File
@@ -0,0 +1,39 @@
from __future__ import annotations
from pydantic import Field
from stirling.models import ApiModel
from .common import FileId
class IngestedPageText(ApiModel):
page_number: int = Field(ge=1)
text: str
class IngestDocumentRequest(ApiModel):
"""Replace-ingest a document's content into RAG under the given document_id.
Each content-type field is optional; the endpoint replaces the document's entire
stored content with whatever is provided. To add a content type later, call again
with all content types the document should have (incremental-add-without-replace
will be a separate endpoint if/when we need it).
``source`` is a human-readable label (typically the original filename) that flows
into chunk metadata so search results are readable when document_id is a hash.
"""
document_id: FileId = Field(min_length=1)
source: str = Field(min_length=1)
page_text: list[IngestedPageText] | None = None
class IngestDocumentResponse(ApiModel):
document_id: FileId
chunks_indexed: int
class DeleteDocumentResponse(ApiModel):
document_id: FileId
deleted: bool
-20
View File
@@ -1,20 +0,0 @@
from __future__ import annotations
from stirling.documents.embedder import EmbeddingService
from stirling.documents.pgvector_store import PgVectorStore
from stirling.documents.rag_capability import RagCapability
from stirling.documents.service import DocumentService
from stirling.documents.sqlite_vec_store import SqliteVecStore
from stirling.documents.store import Document, DocumentStore, SearchResult, StoredPage
__all__ = [
"Document",
"DocumentService",
"DocumentStore",
"EmbeddingService",
"PgVectorStore",
"RagCapability",
"SearchResult",
"SqliteVecStore",
"StoredPage",
]
-133
View File
@@ -1,133 +0,0 @@
from __future__ import annotations
import logging
from stirling.contracts.documents import Page, PageRange, PageText
from stirling.documents.embedder import EmbeddingService
from stirling.documents.store import Document, DocumentStore, SearchResult, StoredPage
from stirling.models import FileId
logger = logging.getLogger(__name__)
PAGE_NUMBER_METADATA_KEY = "page_number"
CONTENT_TYPE_METADATA_KEY = "content_type"
PAGE_TEXT_CONTENT_TYPE = "page_text"
class DocumentService:
"""Top-level facade for stored document content.
Holds two representations of every document under a single ``collection``:
* **Vector chunks** for RAG-style semantic retrieval (``search``).
* **Ordered pages** for whole-document reading (``read_pages``).
Both are populated by :meth:`ingest` from a single ``pages`` payload. Agents
pick the strategy that fits the question; they don't need to know which
storage they're hitting.
"""
def __init__(self, embedder: EmbeddingService, store: DocumentStore, default_top_k: int = 5) -> None:
self._embedder = embedder
self._store = store
self._default_top_k = default_top_k
async def ingest(
self,
collection: FileId,
pages: list[PageText],
source: str,
) -> int:
"""Replace-ingest a document. Returns the number of vector chunks indexed.
This wipes any previously-stored content for ``collection`` and writes
both the vector-chunk and page-text representations from the same
``pages`` payload. Pages with empty/whitespace-only text are skipped
for chunking but still written to the page store so page numbering is
preserved end-to-end.
"""
await self._store.delete_collection(collection)
await self._store.ensure_collection(collection, source)
stored_pages = [StoredPage(page_number=p.page_number, text=p.text, char_count=len(p.text)) for p in pages]
await self._store.add_pages(collection, stored_pages)
chunks: list[Document] = []
for page in pages:
if not page.text.strip():
continue
chunks.extend(
self._embedder.chunk_and_prepare(
text=page.text,
source=f"{source}:page:{page.page_number}",
base_metadata={
PAGE_NUMBER_METADATA_KEY: str(page.page_number),
CONTENT_TYPE_METADATA_KEY: PAGE_TEXT_CONTENT_TYPE,
},
)
)
if not chunks:
return 0
embeddings = await self._embedder.embed_documents([doc.text for doc in chunks])
await self._store.add_documents(collection, chunks, embeddings)
return len(chunks)
async def search(
self,
query: str,
collection: FileId | None = None,
top_k: int | None = None,
) -> list[SearchResult]:
"""Embed query and search across one or all collections.
If collection is None, searches all available collections and merges results.
"""
k = top_k if top_k is not None else self._default_top_k
query_embedding = await self._embedder.embed_query(query)
if collection is not None:
if not await self._store.has_collection(collection):
return []
return await self._store.search(collection, query_embedding, k)
# Search all collections, skipping any that error (e.g. dimension mismatch)
collections = await self._store.list_collections()
all_results: list[SearchResult] = []
for col_name in collections:
try:
results = await self._store.search(col_name, query_embedding, k)
all_results.extend(results)
except Exception: # noqa: BLE001 - any backend error on one collection should not stop the others
logger.warning("Skipping collection %s during cross-collection search", col_name, exc_info=True)
# Sort by score descending, return top_k across all collections
all_results.sort(key=lambda r: r.score, reverse=True)
return all_results[:k]
async def read_pages(
self,
collection: FileId,
page_range: PageRange | None = None,
) -> list[Page]:
"""Return ordered page text for ``collection``.
Empty list if the collection has no stored pages.
"""
return await self._store.read_pages(collection, page_range)
async def delete_collection(self, collection: FileId) -> None:
"""Remove a collection's chunks and pages."""
await self._store.delete_collection(collection)
async def has_collection(self, collection: FileId) -> bool:
"""Check whether a collection exists."""
return await self._store.has_collection(collection)
async def list_collections(self) -> list[FileId]:
"""List all available collections."""
return [FileId(name) for name in await self._store.list_collections()]
async def close(self) -> None:
"""Release the underlying store's resources."""
await self._store.close()
-113
View File
@@ -1,113 +0,0 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from stirling.contracts.documents import Page, PageRange
@dataclass
class Document:
"""A chunk of text with metadata, ready for embedding and storage."""
id: str
text: str
metadata: dict[str, str] = field(default_factory=dict)
@dataclass
class SearchResult:
"""A document returned from a vector search with its relevance score."""
document: Document
score: float
@dataclass
class StoredPage:
"""A page as written to the store. ``char_count`` is precomputed at ingest."""
page_number: int
text: str
char_count: int
class DocumentStore(ABC):
"""Abstract interface for document storage backends.
Backends hold two representations of every document:
* **Vector chunks** - small, embedded chunks used for RAG search.
* **Ordered pages** - the original page text retained in document order,
used for whole-document reading.
Both representations live under the same ``collection`` (file id) and are
rooted at a single parent row in ``documents_meta``. Removing that parent
row cascades to both child representations, so :meth:`delete_collection`
is one logical delete.
"""
@abstractmethod
async def ensure_collection(self, collection: str, source: str) -> None:
"""Upsert the top-level ``documents_meta`` row for this collection.
Must be called before :meth:`add_pages` or :meth:`add_documents`. Both
of those write into child tables that hold a foreign key to the parent
row, so it must exist first.
"""
@abstractmethod
async def add_documents(
self,
collection: str,
documents: list[Document],
embeddings: list[list[float]],
) -> None:
"""Store vector chunks with their embeddings in the named collection."""
@abstractmethod
async def search(
self,
collection: str,
query_embedding: list[float],
top_k: int = 5,
) -> list[SearchResult]:
"""Return the top_k most similar vector chunks from the collection."""
@abstractmethod
async def add_pages(self, collection: str, pages: list[StoredPage]) -> None:
"""Replace the stored pages for ``collection`` with the supplied pages.
Implementations must remove any previously-stored pages for the
collection before writing, so callers can re-ingest by calling this
method again.
"""
@abstractmethod
async def read_pages(
self,
collection: str,
page_range: PageRange | None = None,
) -> list[Page]:
"""Return ordered pages for ``collection``.
If ``page_range`` is ``None`` all pages are returned. Otherwise only
pages whose ``page_number`` falls within the inclusive range are
returned. Pages are always ordered by ``page_number`` ascending.
"""
@abstractmethod
async def delete_collection(self, collection: str) -> None:
"""Remove a collection's chunks and pages."""
@abstractmethod
async def list_collections(self) -> list[str]:
"""Return names of all existing collections."""
@abstractmethod
async def has_collection(self, collection: str) -> bool:
"""Check whether a collection exists."""
@abstractmethod
async def close(self) -> None:
"""Release any resources held by the store (connections, handles, etc.)."""
-3
View File
@@ -12,12 +12,9 @@ FileId = NewType("FileId", str)
class ApiModel(BaseModel):
"""Base for every contract model crossing a service boundary."""
model_config = ConfigDict(
alias_generator=to_camel,
extra="forbid",
validate_by_name=True,
validate_by_alias=True,
serialize_by_alias=True,
)
File diff suppressed because it is too large Load Diff
@@ -1,14 +1,4 @@
# Document Storage
The `documents` package owns all stored content for a document under a single
`collection` (file id):
* **Vector chunks** — small, embedded chunks for RAG-style retrieval.
* **Ordered pages** — the original page text retained in document order, used
for whole-document reading.
Both representations are populated by a single `ingest()` call and removed
together by `delete_collection()`.
# RAG Integration Guide
## Adding RAG to an Agent
@@ -32,21 +22,19 @@ That's it. The agent gets a `search_knowledge` tool it can call autonomously.
## Scoping to Specific Collections
Collections are named buckets of indexed documents - think folders. By default
an agent searches everything in the store. Pass `collections=` to restrict it
to only the docs indexed under those names.
Collections are named buckets of indexed documents — think folders. By default an agent searches everything in the store. Pass `collections=` to restrict it to only the docs indexed under those names.
```python
from stirling.documents import RagCapability
from stirling.rag import RagCapability
# Only searches docs indexed under "company-docs"
scoped = RagCapability(runtime.documents, collections=["company-docs"], top_k=3)
# Only searches docs indexed under "company-docs" — ignores everything else
scoped = RagCapability(runtime.rag_service, collections=["company-docs"], top_k=3)
# Searches multiple collections
multi = RagCapability(runtime.documents, collections=["company-docs", "product-specs"])
multi = RagCapability(runtime.rag_service, collections=["company-docs", "product-specs"])
# No collections arg = searches all collections in the store
everything = RagCapability(runtime.documents)
everything = RagCapability(runtime.rag_service)
```
## Config
@@ -63,8 +51,7 @@ STIRLING_RAG_CHUNK_OVERLAP=64
STIRLING_RAG_TOP_K=5
```
Provider credentials (and any local overrides) go in the uncommitted
`engine/.env.local`:
Provider credentials (and any local overrides) go in the uncommitted `engine/.env.local`:
```
VOYAGE_API_KEY=your-key
@@ -72,17 +59,13 @@ VOYAGE_API_KEY=your-key
## Backends
**`sqlite`** - Embedded sqlite-vec. Single `.db` file, zero ops. Ideal for dev
and self-hosted deployments.
**`sqlite`** — Embedded sqlite-vec. Single `.db` file, zero ops. Ideal for dev and self-hosted deployments.
**`pgvector`** - External PostgreSQL with the `vector` extension. Point
`STIRLING_RAG_PGVECTOR_DSN` at your Postgres instance.
**`pgvector`** — External PostgreSQL with the `vector` extension. Point `STIRLING_RAG_PGVECTOR_DSN` at your Postgres instance.
Both backends implement the same `DocumentStore` interface, so agents and the
service work identically regardless of which you pick.
Both backends implement the same `VectorStore` interface, so agents and the RAG service work identically regardless of which you pick.
For a self-hosted embedding server (e.g. Ollama, TEI, vLLM) set the model
string accordingly and point at the server via its native env var:
For a self-hosted embedding server (e.g. Ollama, TEI, vLLM) set the model string accordingly and point at the server via its native env var:
```
# Ollama running on another machine
@@ -98,5 +81,8 @@ OPENAI_BASE_URL=http://192.168.1.50:8080/v1
| Method | Endpoint | Purpose |
|--------|----------|---------|
| POST | `/api/v1/documents` | Replace-ingest a document's pages |
| DELETE | `/api/v1/documents/{document_id}` | Delete a document's stored content |
| GET | `/api/v1/rag/status` | Report embedding model and existing collections |
| POST | `/api/v1/rag/index` | Index text into a collection |
| POST | `/api/v1/rag/search` | Search a collection |
| GET | `/api/v1/rag/collections` | List collections |
| DELETE | `/api/v1/rag/collections/{name}` | Delete a collection |
+19
View File
@@ -0,0 +1,19 @@
from __future__ import annotations
from stirling.rag.capability import RagCapability
from stirling.rag.embedder import EmbeddingService
from stirling.rag.pgvector_store import PgVectorStore
from stirling.rag.service import RagService
from stirling.rag.sqlite_vec_store import SqliteVecStore
from stirling.rag.store import Document, SearchResult, VectorStore
__all__ = [
"Document",
"EmbeddingService",
"PgVectorStore",
"RagCapability",
"RagService",
"SearchResult",
"SqliteVecStore",
"VectorStore",
]
@@ -6,9 +6,9 @@ from collections.abc import Awaitable, Callable
from pydantic_ai import FunctionToolset, RunContext, ToolDefinition
from pydantic_ai.toolsets import AbstractToolset
from stirling.documents.service import DocumentService
from stirling.documents.store import SearchResult
from stirling.models import FileId
from stirling.rag.service import RagService
from stirling.rag.store import SearchResult
logger = logging.getLogger(__name__)
@@ -34,12 +34,12 @@ class RagCapability:
def __init__(
self,
documents: DocumentService,
rag_service: RagService,
collections: list[FileId] | None = None,
top_k: int = 5,
max_searches: int = 5,
) -> None:
self._documents = documents
self._rag_service = rag_service
self._collections = collections
self._top_k = top_k
self._max_searches = max_searches
@@ -74,7 +74,7 @@ class RagCapability:
)
async def _dynamic_instructions(self) -> str:
collections = await self._documents.list_collections()
collections = await self._rag_service.list_collections()
if collections:
names = ", ".join(collections)
collection_desc = f"the following knowledge base collections: {names}"
@@ -115,12 +115,12 @@ class RagCapability:
if self._collections:
all_results = []
for col in self._collections:
col_results = await self._documents.search(query, collection=col, top_k=k)
col_results = await self._rag_service.search(query, collection=col, top_k=k)
all_results.extend(col_results)
all_results.sort(key=lambda r: r.score, reverse=True)
results = all_results[:k]
else:
results = await self._documents.search(query, top_k=k)
results = await self._rag_service.search(query, top_k=k)
if not results:
logger.info("[rag] search_knowledge query=%r -> 0 results", query)
@@ -2,8 +2,8 @@ from __future__ import annotations
from pydantic_ai import Embedder
from stirling.documents.chunker import chunk_text
from stirling.documents.store import Document
from stirling.rag.chunker import chunk_text
from stirling.rag.store import Document
# Keep each upstream embed request under every major provider's per-call limit while
# still batching large enough that a book-sized document ingests in a reasonable number
@@ -5,20 +5,14 @@ import json
import psycopg
from pgvector.psycopg import register_vector_async
from stirling.contracts.documents import Page, PageRange
from stirling.documents.store import Document, DocumentStore, SearchResult, StoredPage
from stirling.rag.store import Document, SearchResult, VectorStore
class PgVectorStore(DocumentStore):
class PgVectorStore(VectorStore):
"""PostgreSQL + pgvector backed store.
Connects to an external Postgres instance (DSN provided via config) and uses the
`vector` extension for similarity search. The schema is created on first use.
Holds two tables under the same connection:
* ``rag_documents`` - vector chunks for RAG search.
* ``document_pages`` - ordered page text for whole-document reading.
"""
def __init__(self, dsn: str) -> None:
@@ -38,20 +32,11 @@ class PgVectorStore(DocumentStore):
async with await self._connect() as conn:
async with conn.cursor() as cur:
await cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
await cur.execute(
"""
CREATE TABLE IF NOT EXISTS documents_meta (
collection TEXT PRIMARY KEY,
source TEXT NOT NULL
)
"""
)
await cur.execute(
"""
CREATE TABLE IF NOT EXISTS rag_documents (
id TEXT NOT NULL,
collection TEXT NOT NULL
REFERENCES documents_meta(collection) ON DELETE CASCADE,
collection TEXT NOT NULL,
text TEXT NOT NULL,
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
embedding vector NOT NULL,
@@ -60,36 +45,9 @@ class PgVectorStore(DocumentStore):
"""
)
await cur.execute("CREATE INDEX IF NOT EXISTS idx_rag_collection ON rag_documents(collection)")
await cur.execute(
"""
CREATE TABLE IF NOT EXISTS document_pages (
collection TEXT NOT NULL
REFERENCES documents_meta(collection) ON DELETE CASCADE,
page_number INTEGER NOT NULL,
text TEXT NOT NULL,
char_count INTEGER NOT NULL,
PRIMARY KEY (collection, page_number)
)
"""
)
await cur.execute("CREATE INDEX IF NOT EXISTS idx_pages_collection ON document_pages(collection)")
await conn.commit()
self._initialized = True
async def ensure_collection(self, collection: str, source: str) -> None:
await self._ensure_schema()
async with await self._connect() as conn:
async with conn.cursor() as cur:
await cur.execute(
"""
INSERT INTO documents_meta (collection, source)
VALUES (%s, %s)
ON CONFLICT (collection) DO UPDATE SET source = EXCLUDED.source
""",
(collection, source),
)
await conn.commit()
async def add_documents(
self,
collection: str,
@@ -148,58 +106,18 @@ class PgVectorStore(DocumentStore):
for r in rows
]
async def add_pages(self, collection: str, pages: list[StoredPage]) -> None:
await self._ensure_schema()
async with await self._connect() as conn:
async with conn.cursor() as cur:
await cur.execute("DELETE FROM document_pages WHERE collection = %s", (collection,))
if pages:
await cur.executemany(
"""
INSERT INTO document_pages (collection, page_number, text, char_count)
VALUES (%s, %s, %s, %s)
""",
[(collection, p.page_number, p.text, p.char_count) for p in pages],
)
await conn.commit()
async def read_pages(
self,
collection: str,
page_range: PageRange | None = None,
) -> list[Page]:
await self._ensure_schema()
async with await self._connect() as conn:
async with conn.cursor() as cur:
if page_range is None:
await cur.execute(
"SELECT page_number, text, char_count FROM document_pages "
"WHERE collection = %s ORDER BY page_number",
(collection,),
)
else:
await cur.execute(
"SELECT page_number, text, char_count FROM document_pages "
"WHERE collection = %s AND page_number BETWEEN %s AND %s "
"ORDER BY page_number",
(collection, page_range.start, page_range.end),
)
rows = await cur.fetchall()
return [Page(page_number=r[0], text=r[1], char_count=r[2]) for r in rows]
async def delete_collection(self, collection: str) -> None:
await self._ensure_schema()
async with await self._connect() as conn:
async with conn.cursor() as cur:
# Cascade FKs handle rag_documents and document_pages.
await cur.execute("DELETE FROM documents_meta WHERE collection = %s", (collection,))
await cur.execute("DELETE FROM rag_documents WHERE collection = %s", (collection,))
await conn.commit()
async def list_collections(self) -> list[str]:
await self._ensure_schema()
async with await self._connect() as conn:
async with conn.cursor() as cur:
await cur.execute("SELECT collection FROM documents_meta ORDER BY collection")
await cur.execute("SELECT DISTINCT collection FROM rag_documents ORDER BY collection")
rows = await cur.fetchall()
return [r[0] for r in rows]
@@ -208,7 +126,7 @@ class PgVectorStore(DocumentStore):
async with await self._connect() as conn:
async with conn.cursor() as cur:
await cur.execute(
"SELECT 1 FROM documents_meta WHERE collection = %s",
"SELECT 1 FROM rag_documents WHERE collection = %s LIMIT 1",
(collection,),
)
row = await cur.fetchone()
+102
View File
@@ -0,0 +1,102 @@
from __future__ import annotations
import logging
from stirling.models import FileId
from stirling.rag.embedder import EmbeddingService
from stirling.rag.store import Document, SearchResult, VectorStore
logger = logging.getLogger(__name__)
class RagService:
"""Orchestrates embedding and vector storage for RAG workflows."""
def __init__(self, embedder: EmbeddingService, store: VectorStore, default_top_k: int = 5) -> None:
self._embedder = embedder
self._store = store
self._default_top_k = default_top_k
async def index_text(
self,
collection: FileId,
text: str,
source: str = "",
metadata: dict[str, str] | None = None,
) -> int:
"""Chunk, embed, and store text. Returns the number of chunks indexed."""
documents = self._embedder.chunk_and_prepare(text, source=source, base_metadata=metadata)
if not documents:
return 0
embeddings = await self._embedder.embed_documents([doc.text for doc in documents])
await self._store.add_documents(collection, documents, embeddings)
return len(documents)
async def index_documents(self, collection: FileId, documents: list[Document]) -> int:
"""Embed and store pre-chunked documents. Returns the number stored."""
if not documents:
return 0
embeddings = await self._embedder.embed_documents([doc.text for doc in documents])
await self._store.add_documents(collection, documents, embeddings)
return len(documents)
def chunk_text(
self,
text: str,
source: str = "",
base_metadata: dict[str, str] | None = None,
) -> list[Document]:
"""Chunk text into Document objects ready for indexing. Does NOT embed.
Exposed so callers that ingest many chunks can accumulate them across calls
and then pass the full batch to ``index_documents`` for a single embedding pass.
"""
return self._embedder.chunk_and_prepare(text, source=source, base_metadata=base_metadata)
async def search(
self,
query: str,
collection: FileId | None = None,
top_k: int | None = None,
) -> list[SearchResult]:
"""Embed query and search across one or all collections.
If collection is None, searches all available collections and merges results.
"""
k = top_k if top_k is not None else self._default_top_k
query_embedding = await self._embedder.embed_query(query)
if collection is not None:
if not await self._store.has_collection(collection):
return []
return await self._store.search(collection, query_embedding, k)
# Search all collections, skipping any that error (e.g. dimension mismatch)
collections = await self._store.list_collections()
all_results: list[SearchResult] = []
for col_name in collections:
try:
results = await self._store.search(col_name, query_embedding, k)
all_results.extend(results)
except Exception: # noqa: BLE001 — any backend error on one collection should not stop the others
logger.warning("Skipping collection %s during cross-collection search", col_name, exc_info=True)
# Sort by score descending, return top_k across all collections
all_results.sort(key=lambda r: r.score, reverse=True)
return all_results[:k]
async def delete_collection(self, collection: FileId) -> None:
"""Remove a collection and all its documents."""
await self._store.delete_collection(collection)
async def has_collection(self, collection: FileId) -> bool:
"""Check whether a collection exists."""
return await self._store.has_collection(collection)
async def list_collections(self) -> list[FileId]:
"""List all available collections."""
return [FileId(name) for name in await self._store.list_collections()]
async def close(self) -> None:
"""Release the underlying vector store's resources."""
await self._store.close()
@@ -9,11 +9,10 @@ from pathlib import Path
import sqlite_vec
from stirling.contracts.documents import Page, PageRange
from stirling.documents.store import Document, DocumentStore, SearchResult, StoredPage
from stirling.rag.store import Document, SearchResult, VectorStore
class SqliteVecStore(DocumentStore):
class SqliteVecStore(VectorStore):
"""sqlite-vec backed vector store. Single-file SQLite database, embedded, no server.
Each collection gets its own `vec0` virtual table with a fixed embedding dimension
@@ -33,8 +32,6 @@ class SqliteVecStore(DocumentStore):
conn.enable_load_extension(True)
sqlite_vec.load(conn)
conn.enable_load_extension(False)
# Required so cascade deletes from documents_meta clean up child tables.
conn.execute("PRAGMA foreign_keys=ON")
if self._db_path is not None:
conn.execute("PRAGMA journal_mode=WAL")
@@ -48,18 +45,10 @@ class SqliteVecStore(DocumentStore):
return cls(":memory:")
def _init_schema(self) -> None:
self._conn.execute(
"""
CREATE TABLE IF NOT EXISTS documents_meta (
collection TEXT PRIMARY KEY,
source TEXT NOT NULL
)
"""
)
self._conn.execute(
"""
CREATE TABLE IF NOT EXISTS collections (
name TEXT PRIMARY KEY REFERENCES documents_meta(collection) ON DELETE CASCADE,
name TEXT PRIMARY KEY,
dim INTEGER NOT NULL,
table_name TEXT NOT NULL
)
@@ -69,7 +58,7 @@ class SqliteVecStore(DocumentStore):
"""
CREATE TABLE IF NOT EXISTS documents (
id TEXT NOT NULL,
collection TEXT NOT NULL REFERENCES documents_meta(collection) ON DELETE CASCADE,
collection TEXT NOT NULL,
text TEXT NOT NULL,
metadata TEXT NOT NULL DEFAULT '{}',
vec_rowid INTEGER NOT NULL,
@@ -78,32 +67,6 @@ class SqliteVecStore(DocumentStore):
"""
)
self._conn.execute("CREATE INDEX IF NOT EXISTS idx_doc_collection ON documents(collection)")
self._conn.execute(
"""
CREATE TABLE IF NOT EXISTS document_pages (
collection TEXT NOT NULL REFERENCES documents_meta(collection) ON DELETE CASCADE,
page_number INTEGER NOT NULL,
text TEXT NOT NULL,
char_count INTEGER NOT NULL,
PRIMARY KEY (collection, page_number)
)
"""
)
self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_collection ON document_pages(collection)")
self._conn.commit()
async def ensure_collection(self, collection: str, source: str) -> None:
async with self._lock:
await asyncio.to_thread(self._sync_ensure_collection, collection, source)
def _sync_ensure_collection(self, collection: str, source: str) -> None:
self._conn.execute(
"""
INSERT INTO documents_meta(collection, source) VALUES (?, ?)
ON CONFLICT(collection) DO UPDATE SET source = excluded.source
""",
(collection, source),
)
self._conn.commit()
@staticmethod
@@ -233,56 +196,18 @@ class SqliteVecStore(DocumentStore):
for r in results
]
async def add_pages(self, collection: str, pages: list[StoredPage]) -> None:
async with self._lock:
await asyncio.to_thread(self._sync_add_pages, collection, pages)
def _sync_add_pages(self, collection: str, pages: list[StoredPage]) -> None:
self._conn.execute("DELETE FROM document_pages WHERE collection = ?", (collection,))
if pages:
self._conn.executemany(
"INSERT INTO document_pages(collection, page_number, text, char_count) VALUES (?, ?, ?, ?)",
[(collection, p.page_number, p.text, p.char_count) for p in pages],
)
self._conn.commit()
async def read_pages(
self,
collection: str,
page_range: PageRange | None = None,
) -> list[Page]:
async with self._lock:
return await asyncio.to_thread(self._sync_read_pages, collection, page_range)
def _sync_read_pages(
self,
collection: str,
page_range: PageRange | None,
) -> list[Page]:
if page_range is None:
rows = self._conn.execute(
"SELECT page_number, text, char_count FROM document_pages WHERE collection = ? ORDER BY page_number",
(collection,),
).fetchall()
else:
rows = self._conn.execute(
"SELECT page_number, text, char_count FROM document_pages "
"WHERE collection = ? AND page_number BETWEEN ? AND ? ORDER BY page_number",
(collection, page_range.start, page_range.end),
).fetchall()
return [Page(page_number=r[0], text=r[1], char_count=r[2]) for r in rows]
async def delete_collection(self, collection: str) -> None:
async with self._lock:
await asyncio.to_thread(self._sync_delete_collection, collection)
def _sync_delete_collection(self, collection: str) -> None:
# Drop the sqlite-vec virtual table first; FK cascade handles the regular tables
# (collections, documents, document_pages) when documents_meta is deleted.
row = self._conn.execute("SELECT table_name FROM collections WHERE name = ?", (collection,)).fetchone()
if row is not None:
self._conn.execute(f"DROP TABLE IF EXISTS {row[0]}")
self._conn.execute("DELETE FROM documents_meta WHERE collection = ?", (collection,))
if row is None:
return
table_name = row[0]
self._conn.execute(f"DROP TABLE IF EXISTS {table_name}")
self._conn.execute("DELETE FROM documents WHERE collection = ?", (collection,))
self._conn.execute("DELETE FROM collections WHERE name = ?", (collection,))
self._conn.commit()
async def list_collections(self) -> list[str]:
@@ -290,7 +215,7 @@ class SqliteVecStore(DocumentStore):
return await asyncio.to_thread(self._sync_list_collections)
def _sync_list_collections(self) -> list[str]:
rows = self._conn.execute("SELECT collection FROM documents_meta ORDER BY collection").fetchall()
rows = self._conn.execute("SELECT name FROM collections ORDER BY name").fetchall()
return [r[0] for r in rows]
async def has_collection(self, collection: str) -> bool:
@@ -298,10 +223,7 @@ class SqliteVecStore(DocumentStore):
return await asyncio.to_thread(self._sync_has_collection, collection)
def _sync_has_collection(self, collection: str) -> bool:
row = self._conn.execute(
"SELECT 1 FROM documents_meta WHERE collection = ?",
(collection,),
).fetchone()
row = self._conn.execute("SELECT 1 FROM collections WHERE name = ?", (collection,)).fetchone()
return row is not None
async def close(self) -> None:
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
@dataclass
class Document:
"""A chunk of text with metadata, ready for embedding and storage."""
id: str
text: str
metadata: dict[str, str] = field(default_factory=dict)
@dataclass
class SearchResult:
"""A document returned from a vector search with its relevance score."""
document: Document
score: float
class VectorStore(ABC):
"""Abstract interface for vector storage backends.
Implementations must handle persistence, collection management,
and nearest-neighbor search over pre-computed embeddings.
"""
@abstractmethod
async def add_documents(
self,
collection: str,
documents: list[Document],
embeddings: list[list[float]],
) -> None:
"""Store documents with their embeddings in the named collection."""
@abstractmethod
async def search(
self,
collection: str,
query_embedding: list[float],
top_k: int = 5,
) -> list[SearchResult]:
"""Return the top_k most similar documents from the collection."""
@abstractmethod
async def delete_collection(self, collection: str) -> None:
"""Remove a collection and all its documents."""
@abstractmethod
async def list_collections(self) -> list[str]:
"""Return names of all existing collections."""
@abstractmethod
async def has_collection(self, collection: str) -> bool:
"""Check whether a collection exists."""
@abstractmethod
async def close(self) -> None:
"""Release any resources held by the store (connections, handles, etc.)."""
-10
View File
@@ -1,21 +1,11 @@
"""Shared services used by the Stirling AI runtime."""
from .progress import (
ProgressEmitter,
emit_progress,
reset_progress_emitter,
set_progress_emitter,
)
from .runtime import AppRuntime, build_model_settings, build_runtime
from .tracking import setup_posthog_tracking
__all__ = [
"AppRuntime",
"ProgressEmitter",
"build_model_settings",
"build_runtime",
"emit_progress",
"reset_progress_emitter",
"set_progress_emitter",
"setup_posthog_tracking",
]
-47
View File
@@ -1,47 +0,0 @@
"""Per-request progress emission, plumbed via a ContextVar so deep call stacks
can publish typed events to the streaming orchestrator endpoint without every
intermediate layer knowing about it.
Outside a streaming request no emitter is bound and ``emit_progress`` is a
no-op, so callers in agents/services can emit unconditionally.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from contextvars import ContextVar, Token
from stirling.contracts import ProgressEvent
logger = logging.getLogger(__name__)
type ProgressEmitter = Callable[[ProgressEvent], Awaitable[None]]
_emitter: ContextVar[ProgressEmitter | None] = ContextVar("stirling_progress_emitter", default=None)
def set_progress_emitter(emitter: ProgressEmitter | None) -> Token[ProgressEmitter | None]:
return _emitter.set(emitter)
def reset_progress_emitter(token: Token[ProgressEmitter | None]) -> None:
_emitter.reset(token)
async def emit_progress(event: ProgressEvent) -> None:
"""Publish ``event`` to the current request's emitter, if any.
Failures inside the emitter are logged and swallowed so progress emission
can never break the work it's reporting on.
"""
emitter = _emitter.get()
if emitter is None:
return
try:
await emitter(event)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("progress emitter raised; dropping event %r", event.phase)
+18 -54
View File
@@ -4,49 +4,28 @@ import logging
from dataclasses import dataclass
from typing import assert_never
import httpx
from pydantic_ai.models import Model, infer_model
from pydantic_ai.models.anthropic import AnthropicModel
from pydantic_ai.providers.anthropic import AnthropicProvider
from pydantic_ai.settings import ModelSettings
from stirling.config import ENGINE_ROOT, AppSettings, RagBackend
from stirling.documents import (
DocumentService,
DocumentStore,
from stirling.rag import (
EmbeddingService,
PgVectorStore,
RagCapability,
RagService,
SqliteVecStore,
VectorStore,
)
logger = logging.getLogger(__name__)
def _build_anthropic_http_client() -> httpx.AsyncClient:
"""Build the httpx client used for Anthropic API calls.
We disable connection-pool keepalive so every request opens a fresh
TCP+TLS connection. The default HTTP/1.1 pool reuses connections that
Anthropic's front door (Cloudflare) sometimes closes silently between
requests; the next request that picks up a stale connection sends its
body into a black hole and never gets a response, hanging until our
chunked-reasoner timeout fires.
A fresh handshake costs ~150ms — rounding error against a 5-15s LLM
call. The trade is determinism: we never reuse a connection that might
have died in the pool. See ``STIRLING_HTTP_DEBUG`` traces of slice 6
on 2026-05-06 for the concrete failure mode this addresses.
"""
return httpx.AsyncClient(limits=httpx.Limits(max_keepalive_connections=0))
@dataclass(frozen=True)
class AppRuntime:
settings: AppSettings
fast_model: Model
smart_model: Model
documents: DocumentService
rag_service: RagService
rag_capability: RagCapability
@property
@@ -74,62 +53,47 @@ def validate_structured_output_support(model: Model, model_name: str) -> None:
raise ValueError(f"Unsupported model {model_name}. This model does not support structured outputs.")
def _build_document_store(settings: AppSettings) -> DocumentStore:
"""Build the configured document store backend."""
def _build_vector_store(settings: AppSettings) -> VectorStore:
"""Build the configured vector store backend."""
if settings.rag_backend == RagBackend.SQLITE:
store_path = settings.rag_store_path
# Treat ":memory:" as a special in-process token; otherwise resolve against the engine root.
if str(store_path) != ":memory:" and not store_path.is_absolute():
store_path = ENGINE_ROOT / store_path
logger.info("Document store backend=sqlite, db_path=%s", store_path)
logger.info("RAG backend=sqlite, db_path=%s", store_path)
return SqliteVecStore(db_path=store_path)
if settings.rag_backend == RagBackend.PGVECTOR:
logger.info("Document store backend=pgvector, dsn=<configured>")
logger.info("RAG backend=pgvector, dsn=<configured>")
return PgVectorStore(dsn=settings.rag_pgvector_dsn)
assert_never(settings.rag_backend)
def _build_documents(settings: AppSettings) -> tuple[DocumentService, RagCapability]:
"""Build the document service and the RAG-search capability that wraps it."""
logger.info("Documents: embedding_model=%s", settings.rag_embedding_model)
def _build_rag(settings: AppSettings) -> tuple[RagService, RagCapability]:
"""Build the RAG service and capability."""
logger.info("RAG: embedding_model=%s", settings.rag_embedding_model)
embedder = EmbeddingService(
model_name=settings.rag_embedding_model,
chunk_size=settings.rag_chunk_size,
chunk_overlap=settings.rag_chunk_overlap,
)
store = _build_document_store(settings)
service = DocumentService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k)
capability = RagCapability(documents=service, top_k=settings.rag_default_top_k)
store = _build_vector_store(settings)
service = RagService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k)
capability = RagCapability(rag_service=service, top_k=settings.rag_default_top_k)
return service, capability
def build_runtime(settings: AppSettings) -> AppRuntime:
fast_model = _build_model(settings.fast_model_name)
smart_model = _build_model(settings.smart_model_name)
fast_model = infer_model(settings.fast_model_name)
smart_model = infer_model(settings.smart_model_name)
validate_structured_output_support(fast_model, settings.fast_model_name)
validate_structured_output_support(smart_model, settings.smart_model_name)
documents, rag_capability = _build_documents(settings)
rag_service, rag_capability = _build_rag(settings)
return AppRuntime(
settings=settings,
fast_model=fast_model,
smart_model=smart_model,
documents=documents,
rag_service=rag_service,
rag_capability=rag_capability,
)
def _build_model(model_name: str) -> Model:
"""Construct a model, injecting our keepalive-free httpx client for
Anthropic models so workers don't pick up stale pooled connections.
Other providers fall back to ``infer_model`` defaults; the stale-pool
issue is specific to the Cloudflare-fronted Anthropic API in our
observations and the fix doesn't necessarily apply elsewhere.
"""
if model_name.startswith("anthropic:"):
bare_name = model_name.removeprefix("anthropic:")
provider = AnthropicProvider(http_client=_build_anthropic_http_client())
return AnthropicModel(bare_name, provider=provider)
return infer_model(model_name)
@@ -1,511 +0,0 @@
"""Tests for ``ChunkedReasoner``: chunking, fan-out, and synthesis wiring.
LLM calls are stubbed at the agent boundary; the runtime fixture supplies a
``test`` model so construction succeeds without provider credentials.
"""
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from unittest.mock import AsyncMock, patch
import pytest
from pydantic import BaseModel
from stirling.agents.shared.chunked_reasoner import ChunkedReasoner, ChunkNotes
from stirling.contracts import WholeDocSliceDone
from stirling.contracts.documents import Page
from stirling.services import reset_progress_emitter, set_progress_emitter
from stirling.services.runtime import AppRuntime
@dataclass
class _StubAgentResult[T]:
output: T
class _Answer(BaseModel):
answer: str
def _page(n: int, text: str) -> Page:
return Page(page_number=n, text=text, char_count=len(text))
# Slicing logic
class TestSlicePages:
"""``_slice_pages`` is pure: no model calls, no I/O."""
def test_groups_consecutive_pages_under_budget(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=20)
pages = [_page(1, "abc"), _page(2, "defgh"), _page(3, "ij")]
slices = reasoner._slice_pages(pages)
assert len(slices) == 1
assert [p.page_number for p in slices[0]] == [1, 2, 3]
def test_starts_new_slice_when_budget_exceeded(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=10)
pages = [_page(1, "a" * 6), _page(2, "b" * 6), _page(3, "c" * 6)]
slices = reasoner._slice_pages(pages)
# Each page is 6 chars, budget 10 -> two pages would be 12 (over),
# so the slicer breaks after each page.
assert [[p.page_number for p in s] for s in slices] == [[1], [2], [3]]
def test_oversized_page_becomes_its_own_slice(self, runtime: AppRuntime) -> None:
"""A single page larger than the budget is never split. The reasoner
accepts that this slice exceeds the budget rather than breaking page
boundaries."""
reasoner = ChunkedReasoner(runtime, chars_per_slice=10)
pages = [_page(1, "small"), _page(2, "x" * 100), _page(3, "tiny")]
slices = reasoner._slice_pages(pages)
assert [[p.page_number for p in s] for s in slices] == [[1], [2], [3]]
def test_preserves_input_order(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=1000)
pages = [_page(i, f"page-{i}") for i in range(1, 11)]
slices = reasoner._slice_pages(pages)
flattened = [p.page_number for s in slices for p in s]
assert flattened == list(range(1, 11))
# End-to-end orchestration
class TestReason:
@pytest.mark.anyio
async def test_runs_one_chunk_per_slice_and_synthesises(self, runtime: AppRuntime) -> None:
"""Three small pages with a generous budget produce one chunk and one extractor call;
the synthesis stage receives notes from all chunks and returns the final answer."""
reasoner = ChunkedReasoner(runtime, chars_per_slice=1000)
pages = [_page(1, "alpha"), _page(2, "beta"), _page(3, "gamma")]
canned_notes = ChunkNotes(pages=[1, 2, 3], summary="all three pages", facts=["fact-1"])
canned_answer = _Answer(answer="final answer")
with (
patch.object(reasoner, "_extract_chunk", AsyncMock(return_value=(canned_notes, 0.0))) as chunk_mock,
patch.object(reasoner, "_synthesise", AsyncMock(return_value=canned_answer)) as synth_mock,
):
result = await reasoner.reason(
pages=pages,
question="summarise this",
answer_prompt="answer the question from the notes",
answer_type=_Answer,
)
assert result == canned_answer
assert chunk_mock.await_count == 1
synth_mock.assert_awaited_once()
synth_args = synth_mock.await_args
assert synth_args is not None
# _synthesise(question, notes, answer_prompt, answer_type)
_, notes_arg, _, type_arg = synth_args.args
assert notes_arg == [canned_notes]
assert type_arg is _Answer
@pytest.mark.anyio
async def test_fans_out_when_pages_exceed_slice_budget(self, runtime: AppRuntime) -> None:
"""Pages that don't fit into a single slice produce one extractor call per slice."""
reasoner = ChunkedReasoner(runtime, chars_per_slice=10)
pages = [_page(i, "x" * 8) for i in range(1, 6)]
canned_notes = ChunkNotes(pages=[0], summary="placeholder")
canned_answer = _Answer(answer="ok")
with (
patch.object(reasoner, "_extract_chunk", AsyncMock(return_value=(canned_notes, 0.0))) as chunk_mock,
patch.object(reasoner, "_synthesise", AsyncMock(return_value=canned_answer)),
):
await reasoner.reason(
pages=pages,
question="aggregate",
answer_prompt="answer",
answer_type=_Answer,
)
# 5 pages, budget 10, each page 8 chars -> 5 slices -> 5 chunk calls.
assert chunk_mock.await_count == 5
@pytest.mark.anyio
async def test_skips_first_round_chunks_that_raise_and_continues(self, runtime: AppRuntime) -> None:
"""First-round chunks have no fallback notes, so a failure is dropped
rather than preserving anything; the surviving notes still flow into
synthesis."""
reasoner = ChunkedReasoner(runtime, chars_per_slice=10)
pages = [_page(i, "x" * 8) for i in range(1, 4)]
good = ChunkNotes(pages=[1], summary="ok")
async_results = [good, RuntimeError("chunk boom"), good]
async def _chunk(*_args: object, **_kwargs: object) -> tuple[ChunkNotes, float]:
value = async_results.pop(0)
if isinstance(value, BaseException):
raise value
return value, 0.0
canned_answer = _Answer(answer="resilient")
with (
patch.object(reasoner, "_extract_chunk", AsyncMock(side_effect=_chunk)),
patch.object(reasoner, "_synthesise", AsyncMock(return_value=canned_answer)) as synth_mock,
):
result = await reasoner.reason(
pages=pages,
question="aggregate",
answer_prompt="answer",
answer_type=_Answer,
)
assert result == canned_answer
synth_args = synth_mock.await_args
assert synth_args is not None
_, notes_arg, _, _ = synth_args.args
assert len(notes_arg) == 2
@pytest.mark.anyio
async def test_raises_when_every_first_round_chunk_fails(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=10)
pages = [_page(i, "x" * 8) for i in range(1, 3)]
with (
patch.object(reasoner, "_extract_chunk", AsyncMock(side_effect=RuntimeError("boom"))),
patch.object(reasoner, "_synthesise", AsyncMock()) as synth_mock,
pytest.raises(RuntimeError, match="no notes"),
):
await reasoner.reason(
pages=pages,
question="anything",
answer_prompt="answer",
answer_type=_Answer,
)
synth_mock.assert_not_awaited()
@pytest.mark.anyio
async def test_rejects_empty_pages(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime)
with pytest.raises(ValueError, match="at least one page"):
await reasoner.reason(
pages=[],
question="x",
answer_prompt="y",
answer_type=_Answer,
)
@pytest.mark.anyio
async def test_progress_events_carry_monotonic_completion_counter(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=10)
pages = [_page(i, "x" * 8) for i in range(1, 4)]
# Each chunk's worker awaits a different release event; we release them in
# reverse order so completion order is the inverse of slice order.
release_events = [asyncio.Event() for _ in pages]
next_call_index = 0
async def _gated_worker(*_args: object, **_kwargs: object) -> _StubAgentResult[ChunkNotes]:
nonlocal next_call_index
this_call = next_call_index
next_call_index += 1
await release_events[this_call].wait()
return _StubAgentResult(output=ChunkNotes(pages=[this_call + 1], summary=f"slice-{this_call}"))
emitted: list[WholeDocSliceDone] = []
async def _capture_emitter(event: object) -> None:
if isinstance(event, WholeDocSliceDone):
emitted.append(event)
async def _release_in_reverse() -> None:
# Wait briefly so all three worker tasks are blocked on their events
# before we start releasing them.
await asyncio.sleep(0)
for ev in reversed(release_events):
ev.set()
# Yield so the just-released worker can run to completion before
# we release the next one — keeps ordering deterministic.
await asyncio.sleep(0)
await asyncio.sleep(0)
token = set_progress_emitter(_capture_emitter)
try:
with patch.object(reasoner._extractor, "run", AsyncMock(side_effect=_gated_worker)):
gather_task = asyncio.create_task(reasoner.gather_notes(pages, "anything"))
await _release_in_reverse()
notes = await gather_task
finally:
reset_progress_emitter(token)
assert len(notes) == 3
assert [event.completed for event in emitted] == [1, 2, 3]
assert all(event.total == 3 for event in emitted)
@pytest.mark.anyio
async def test_worker_timeout_is_terminal_for_the_chunk(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=10, worker_timeout_seconds=0.05)
pages = [_page(1, "x" * 8), _page(2, "y" * 8)]
attempts = 0
async def _hang_forever(*_args: object, **_kwargs: object) -> _StubAgentResult[ChunkNotes]:
nonlocal attempts
attempts += 1
await asyncio.sleep(10)
return _StubAgentResult(output=ChunkNotes(pages=[0], summary="never"))
with (
patch.object(reasoner._extractor, "run", AsyncMock(side_effect=_hang_forever)),
patch.object(reasoner, "_synthesise", AsyncMock()) as synth_mock,
pytest.raises(RuntimeError, match="no notes"),
):
await reasoner.reason(
pages=pages,
question="anything",
answer_prompt="answer",
answer_type=_Answer,
)
# One attempt per slice; no retry path.
assert attempts == len(pages)
synth_mock.assert_not_awaited()
@pytest.mark.anyio
async def test_worker_timeout_drops_stalled_chunks(self, runtime: AppRuntime) -> None:
"""A worker that exceeds ``worker_timeout_seconds`` is abandoned, not awaited.
Without this guard one stuck upstream call would pin gather_notes to its
provider HTTP timeout (~10 minutes), starving the orchestrator request.
"""
reasoner = ChunkedReasoner(runtime, chars_per_slice=10, worker_timeout_seconds=0.05)
pages = [_page(i, "x" * 8) for i in range(1, 4)]
async def _hang(*_args: object, **_kwargs: object) -> _StubAgentResult[ChunkNotes]:
await asyncio.sleep(10)
return _StubAgentResult(output=ChunkNotes(pages=[0], summary="never"))
with (
patch.object(reasoner._extractor, "run", AsyncMock(side_effect=_hang)),
patch.object(reasoner, "_synthesise", AsyncMock()) as synth_mock,
pytest.raises(RuntimeError, match="no notes"),
):
await reasoner.reason(
pages=pages,
question="anything",
answer_prompt="answer",
answer_type=_Answer,
)
synth_mock.assert_not_awaited()
# Prompt construction
class TestPromptConstruction:
def test_extraction_prompt_includes_question_and_page_markers(self, runtime: AppRuntime) -> None:
"""A first-round chunk's content carries ``[Page N]`` markers; the
extraction prompt prepends the user question."""
reasoner = ChunkedReasoner(runtime)
chunk = reasoner._chunk_from_pages([_page(2, "page two body"), _page(3, "page three body")])
prompt = reasoner._build_extraction_prompt(chunk.content, "what is on page two?")
assert "what is on page two?" in prompt
assert "[Page 2]" in prompt
assert "[Page 3]" in prompt
assert "page two body" in prompt
def test_format_notes_groups_by_page_label(self) -> None:
notes = [
ChunkNotes(pages=[1], summary="single", facts=["f-1"]),
ChunkNotes(pages=[2, 3, 4], summary="range", relevant_excerpts=["quote-1"]),
]
rendered = ChunkedReasoner.format_notes(notes)
assert "[Notes from page 1]" in rendered
assert "[Notes from pages 2-4]" in rendered
assert "f-1" in rendered
assert "quote-1" in rendered
# Hierarchical compression
#
# The compression loop is part of ``_run_chunks`` and isn't exposed
# directly, so these tests drive it end-to-end via ``gather_notes`` with a
# stubbed extractor that controls per-call output (and per-call failure
# patterns) by counting calls.
class TestCompression:
@pytest.mark.anyio
async def test_no_compression_when_under_budget(self, runtime: AppRuntime) -> None:
"""First-round notes that already fit the budget result in zero
compression rounds: the only extractor calls are one per slice."""
from stirling.agents.shared.chunked_reasoner import _ExtractedNotes
reasoner = ChunkedReasoner(runtime, chars_per_slice=200, notes_char_budget=10_000)
pages = [_page(i, "x" * 150) for i in range(1, 5)] # each page exceeds slice budget alone -> 4 slices
canned = _ExtractedNotes(summary="ok")
with patch.object(
reasoner._extractor,
"run",
AsyncMock(return_value=_StubAgentResult(output=canned)),
) as ext_mock:
notes = await reasoner.gather_notes(pages, "anything")
assert ext_mock.await_count == 4
assert len(notes) == 4
@pytest.mark.anyio
async def test_runs_compression_when_over_budget(self, runtime: AppRuntime) -> None:
"""When first-round notes overflow the budget, the loop regroups them
and runs the extractor again. Output is shorter than input; pages from
every input slice survive in the consolidated notes."""
from stirling.agents.shared.chunked_reasoner import _ExtractedNotes
reasoner = ChunkedReasoner(runtime, chars_per_slice=200, notes_char_budget=200)
pages = [_page(i, "x" * 150) for i in range(1, 5)]
call_count = 0
async def _stub(*_args: object, **_kwargs: object) -> _StubAgentResult[object]:
nonlocal call_count
call_count += 1
if call_count <= 4:
# Round 1: each note ~80 chars rendered. 4 * 80 = 320 chars, over the 200 budget.
return _StubAgentResult(output=_ExtractedNotes(summary="x" * 60))
# Round 2: smaller note so the post-round set fits the budget.
return _StubAgentResult(output=_ExtractedNotes(summary="ok"))
with patch.object(reasoner._extractor, "run", AsyncMock(side_effect=_stub)) as ext_mock:
notes = await reasoner.gather_notes(pages, "anything")
# 4 first-round + 2 compression-round calls = 6 total.
assert ext_mock.await_count == 6
# Compressed from 4 notes to 2.
assert len(notes) == 2
# Pages from every original slice are preserved through compression.
assert sorted({p for n in notes for p in n.pages}) == [1, 2, 3, 4]
@pytest.mark.anyio
async def test_compression_preserves_input_notes_when_a_group_fails(self, runtime: AppRuntime) -> None:
"""A compression chunk that raises has its input notes carried forward
rather than dropped, so page coverage isn't silently lost. The
succeeding chunk is replaced by its consolidated note.
Budget is sized so the post-round survivors (2 preserved + 1
consolidated) fit, leaving a single compression round as the
observable interaction."""
from stirling.agents.shared.chunked_reasoner import _ExtractedNotes
reasoner = ChunkedReasoner(runtime, chars_per_slice=200, notes_char_budget=300)
pages = [_page(i, "x" * 150) for i in range(1, 5)]
call_count = 0
async def _stub(*_args: object, **_kwargs: object) -> _StubAgentResult[object]:
nonlocal call_count
call_count += 1
if call_count <= 4:
return _StubAgentResult(output=_ExtractedNotes(summary="x" * 60))
if call_count == 5:
# The first compression call (covering 2 of the round-1 notes) fails.
raise RuntimeError("compression group fails")
return _StubAgentResult(output=_ExtractedNotes(summary="ok"))
with patch.object(reasoner._extractor, "run", AsyncMock(side_effect=_stub)):
notes = await reasoner.gather_notes(pages, "anything")
# 2 preserved round-1 notes + 1 consolidated note = 3 notes total. Pages
# from every original slice are still covered (preservation worked).
assert len(notes) == 3
assert sorted({p for n in notes for p in n.pages}) == [1, 2, 3, 4]
consolidated = [n for n in notes if n.summary == "ok"]
preserved = [n for n in notes if n.summary.startswith("x")]
assert len(consolidated) == 1
assert len(preserved) == 2
@pytest.mark.anyio
async def test_compression_bails_when_every_group_fails(self, runtime: AppRuntime) -> None:
"""If every chunk in a compression round fails, every input note is
preserved (none consolidated). The loop exits rather than retrying
the same shape forever."""
from stirling.agents.shared.chunked_reasoner import _ExtractedNotes
reasoner = ChunkedReasoner(runtime, chars_per_slice=200, notes_char_budget=200)
pages = [_page(i, "x" * 150) for i in range(1, 5)]
call_count = 0
async def _stub(*_args: object, **_kwargs: object) -> _StubAgentResult[object]:
nonlocal call_count
call_count += 1
if call_count <= 4:
return _StubAgentResult(output=_ExtractedNotes(summary="x" * 60))
raise RuntimeError("compression always fails")
with patch.object(reasoner._extractor, "run", AsyncMock(side_effect=_stub)):
notes = await reasoner.gather_notes(pages, "anything")
# All 4 round-1 notes preserved through the bailed compression round.
assert len(notes) == 4
assert sorted({p for n in notes for p in n.pages}) == [1, 2, 3, 4]
class TestGroupNotesForCompression:
"""``_group_notes_for_compression`` is pure and packs by rendered char count."""
def test_packs_consecutive_notes_under_budget(self, runtime: AppRuntime) -> None:
reasoner = ChunkedReasoner(runtime, chars_per_slice=10_000)
notes = [ChunkNotes(pages=[i], summary=f"s-{i}") for i in range(1, 5)]
groups = reasoner._group_notes_for_compression(notes)
assert len(groups) == 1
assert [n.pages[0] for n in groups[0]] == [1, 2, 3, 4]
def test_starts_new_group_when_budget_exceeded(self, runtime: AppRuntime) -> None:
"""Each note already exceeds the per-group budget, so each becomes its
own group; this matches how slice-pages handles oversize pages."""
reasoner = ChunkedReasoner(runtime, chars_per_slice=5)
notes = [ChunkNotes(pages=[i], summary=f"slice-{i}-with-prose-to-fill-space") for i in range(1, 5)]
groups = reasoner._group_notes_for_compression(notes)
assert [[n.pages[0] for n in g] for g in groups] == [[1], [2], [3], [4]]
class TestExtractChunk:
@pytest.mark.anyio
async def test_pages_are_unioned_for_compression_chunks(self, runtime: AppRuntime) -> None:
"""A compression chunk's resulting note carries the union of input pages.
The model output schema doesn't include pages, so the wrapper is the
single source of truth."""
from stirling.agents.shared.chunked_reasoner import _ExtractedNotes
reasoner = ChunkedReasoner(runtime)
group = [
ChunkNotes(pages=[1, 2], summary="a"),
ChunkNotes(pages=[3], summary="b"),
ChunkNotes(pages=[4, 5], summary="c"),
]
chunk = reasoner._chunk_from_notes(group)
canned = _ExtractedNotes(summary="merged", facts=["x"], relevant_excerpts=["y"])
with patch.object(
reasoner._extractor,
"run",
AsyncMock(return_value=_StubAgentResult(output=canned)),
):
note, _ = await reasoner._extract_chunk(chunk, "anything")
assert note.pages == [1, 2, 3, 4, 5]
assert note.summary == "merged"
assert note.facts == ["x"]
assert note.relevant_excerpts == ["y"]
@@ -1,220 +0,0 @@
"""Tests for ``WholeDocReaderCapability``: tool dispatch, multi-file iteration,
budget enforcement, and graceful handling of missing pages.
The map-phase LLM call is patched at the reasoner boundary so tests don't hit
any model.
"""
from __future__ import annotations
from dataclasses import replace
from unittest.mock import AsyncMock, patch
import pytest
from stirling.agents.shared import ChunkedReasoner, ChunkNotes, WholeDocReaderCapability
from stirling.contracts import AiFile, PageText
from stirling.documents import Document, DocumentService, SqliteVecStore
from stirling.models import FileId
from stirling.services.runtime import AppRuntime
class StubEmbedder:
"""Deterministic embeddings so tests don't need a real provider."""
def __init__(self, dim: int = 8) -> None:
self._dim = dim
async def embed_query(self, text: str) -> list[float]:
h = hash(text) % 1000
return [(h + i) / 1000.0 for i in range(self._dim)]
async def embed_documents(self, texts: list[str]) -> list[list[float]]:
return [await self.embed_query(t) for t in texts]
def chunk_and_prepare(
self,
text: str,
source: str = "",
base_metadata: dict[str, str] | None = None,
) -> list[Document]:
from stirling.documents.chunker import chunk_text
chunks = chunk_text(text, 100, 10)
docs: list[Document] = []
for i, chunk in enumerate(chunks):
meta = dict(base_metadata) if base_metadata else {}
meta["source"] = source
meta["chunk_index"] = str(i)
doc_id = f"{source}:chunk:{i}" if source else f"chunk:{i}"
docs.append(Document(id=doc_id, text=chunk, metadata=meta))
return docs
@pytest.fixture
def runtime_with_stub_docs(runtime: AppRuntime) -> AppRuntime:
stub = DocumentService(
embedder=StubEmbedder(), # type: ignore[arg-type]
store=SqliteVecStore.ephemeral(),
default_top_k=runtime.settings.rag_default_top_k,
)
return replace(runtime, documents=stub)
def _ai_file(file_id: str, name: str) -> AiFile:
return AiFile(id=FileId(file_id), name=name)
@pytest.mark.anyio
async def test_read_full_document_returns_formatted_notes_for_single_file(
runtime_with_stub_docs: AppRuntime,
) -> None:
"""The tool reads the file's stored pages, calls the reasoner's map phase,
and returns the formatted notes prefixed by the file name."""
pages = [
PageText(page_number=1, text="Chapter one prose."),
PageText(page_number=2, text="Chapter two prose."),
]
await runtime_with_stub_docs.documents.ingest(FileId("doc-id"), pages, source="doc.pdf")
reasoner = ChunkedReasoner(runtime_with_stub_docs)
canned_notes = [ChunkNotes(pages=[1, 2], summary="overview", facts=["fact-A"])]
with patch.object(reasoner, "gather_notes", AsyncMock(return_value=canned_notes)) as gather_mock:
capability = WholeDocReaderCapability(
runtime=runtime_with_stub_docs,
files=[_ai_file("doc-id", "doc.pdf")],
reasoner=reasoner,
)
result = await capability._read_full_document("what is in the document?")
gather_mock.assert_awaited_once()
call = gather_mock.await_args
assert call is not None
pages_arg = call.args[0]
assert [p.page_number for p in pages_arg] == [1, 2]
assert "=== doc.pdf ===" in result
assert "fact-A" in result
assert "[Notes from pages 1-2]" in result
@pytest.mark.anyio
async def test_read_full_document_iterates_multiple_files(runtime_with_stub_docs: AppRuntime) -> None:
"""Multi-file requests run the map phase per file and return one section
per file in the formatted output."""
for cid, source in (("doc-a", "a.pdf"), ("doc-b", "b.pdf")):
await runtime_with_stub_docs.documents.ingest(
FileId(cid),
[PageText(page_number=1, text=f"contents of {cid}")],
source=source,
)
reasoner = ChunkedReasoner(runtime_with_stub_docs)
notes_by_call = [
[ChunkNotes(pages=[1], summary="a-summary")],
[ChunkNotes(pages=[1], summary="b-summary")],
]
async def _gather(*_args: object, **_kwargs: object) -> list[ChunkNotes]:
return notes_by_call.pop(0)
with patch.object(reasoner, "gather_notes", AsyncMock(side_effect=_gather)) as gather_mock:
capability = WholeDocReaderCapability(
runtime=runtime_with_stub_docs,
files=[_ai_file("doc-a", "a.pdf"), _ai_file("doc-b", "b.pdf")],
reasoner=reasoner,
)
result = await capability._read_full_document("compare them")
assert gather_mock.await_count == 2
assert "=== a.pdf ===" in result
assert "a-summary" in result
assert "=== b.pdf ===" in result
assert "b-summary" in result
@pytest.mark.anyio
async def test_read_full_document_skips_files_without_pages(runtime_with_stub_docs: AppRuntime) -> None:
"""Files with no stored pages are quietly skipped; the tool still runs
the map phase for files that do have pages."""
await runtime_with_stub_docs.documents.ingest(
FileId("present"),
[PageText(page_number=1, text="real content")],
source="present.pdf",
)
# 'missing' is never ingested -> read_pages returns [].
reasoner = ChunkedReasoner(runtime_with_stub_docs)
canned = [ChunkNotes(pages=[1], summary="present summary")]
with patch.object(reasoner, "gather_notes", AsyncMock(return_value=canned)) as gather_mock:
capability = WholeDocReaderCapability(
runtime=runtime_with_stub_docs,
files=[_ai_file("missing", "missing.pdf"), _ai_file("present", "present.pdf")],
reasoner=reasoner,
)
result = await capability._read_full_document("anything")
gather_mock.assert_awaited_once()
assert "=== present.pdf ===" in result
assert "missing.pdf" not in result
@pytest.mark.anyio
async def test_read_full_document_returns_empty_message_when_no_pages_anywhere(
runtime_with_stub_docs: AppRuntime,
) -> None:
reasoner = ChunkedReasoner(runtime_with_stub_docs)
with patch.object(reasoner, "gather_notes", AsyncMock()) as gather_mock:
capability = WholeDocReaderCapability(
runtime=runtime_with_stub_docs,
files=[_ai_file("nope", "nope.pdf")],
reasoner=reasoner,
)
result = await capability._read_full_document("anything")
gather_mock.assert_not_awaited()
assert result == "Could not read any document content."
@pytest.mark.anyio
async def test_read_full_document_budget_hides_tool_when_exhausted(
runtime_with_stub_docs: AppRuntime,
) -> None:
"""The prepare callback returns None once max_reads is reached so the
agent can no longer call the tool on subsequent turns. Mirrors
RagCapability's per-run budget."""
await runtime_with_stub_docs.documents.ingest(
FileId("doc-id"),
[PageText(page_number=1, text="content")],
source="doc.pdf",
)
reasoner = ChunkedReasoner(runtime_with_stub_docs)
with patch.object(reasoner, "gather_notes", AsyncMock(return_value=[ChunkNotes(pages=[1], summary="s")])):
capability = WholeDocReaderCapability(
runtime=runtime_with_stub_docs,
files=[_ai_file("doc-id", "doc.pdf")],
reasoner=reasoner,
max_reads=1,
)
sentinel: object = object()
# Budget intact -> prepare returns the tool.
assert await capability._prepare_read_full_document(None, sentinel) is sentinel # type: ignore[arg-type]
# Spend the budget.
await capability._read_full_document("anything")
# Budget spent -> prepare returns None.
assert await capability._prepare_read_full_document(None, sentinel) is None # type: ignore[arg-type]
@pytest.mark.anyio
async def test_instructions_mention_attached_files(runtime_with_stub_docs: AppRuntime) -> None:
capability = WholeDocReaderCapability(
runtime=runtime_with_stub_docs,
files=[_ai_file("doc-a", "alpha.pdf"), _ai_file("doc-b", "beta.pdf")],
)
text = capability.instructions
assert "alpha.pdf" in text
assert "beta.pdf" in text
assert "read_full_document" in text
-4
View File
@@ -31,10 +31,6 @@ def build_app_settings() -> AppSettings:
rag_chunk_overlap=64,
rag_default_top_k=5,
rag_max_searches=5,
chunked_reasoner_chars_per_slice=16_000,
chunked_reasoner_concurrency=10,
chunked_reasoner_notes_char_budget=250_000,
chunked_reasoner_worker_timeout_seconds=60.0,
max_pages=200,
max_characters=200_000,
posthog_enabled=False,
-4
View File
@@ -43,10 +43,6 @@ class StubSettingsProvider:
rag_chunk_overlap=64,
rag_default_top_k=5,
rag_max_searches=5,
chunked_reasoner_chars_per_slice=16_000,
chunked_reasoner_concurrency=10,
chunked_reasoner_worker_timeout_seconds=60.0,
chunked_reasoner_notes_char_budget=250_000,
max_pages=100,
max_characters=100_000,
posthog_enabled=False,
-7
View File
@@ -136,7 +136,6 @@ async def test_pdf_edit_agent_passes_previous_steps_to_parameter_selector(runtim
PdfEditPlanSelection(
operations=[ToolEndpoint.ROTATE_PDF, ToolEndpoint.FLATTEN],
summary="Rotate the PDF, then compress it.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
@@ -295,7 +294,6 @@ async def test_pdf_edit_agent_passes_page_text_to_parameter_selector(runtime: Ap
PdfEditPlanSelection(
operations=[ToolEndpoint.ROTATE_PDF],
summary="Rotate the PDF.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
@@ -392,7 +390,6 @@ async def test_pdf_edit_agent_rejects_plan_referencing_unavailable_operations(
PdfEditPlanSelection(
operations=[ToolEndpoint.COMPRESS_PDF],
summary="Compress.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
@@ -423,7 +420,6 @@ async def test_pdf_edit_agent_supports_literal_find_replace(runtime: AppRuntime)
PdfEditPlanSelection(
operations=[ToolEndpoint.EDIT_TEXT],
summary="Replace 2025 with 2026 throughout the document.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
@@ -470,7 +466,6 @@ async def test_pdf_edit_agent_supports_copy_edit_using_page_text(runtime: AppRun
PdfEditPlanSelection(
operations=[ToolEndpoint.EDIT_TEXT],
summary="Fix typos on page 3.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
@@ -520,7 +515,6 @@ async def test_pdf_edit_agent_supports_natural_language_directed_edit(runtime: A
PdfEditPlanSelection(
operations=[ToolEndpoint.EDIT_TEXT],
summary="Update the company name to Acme Corp.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
@@ -557,7 +551,6 @@ async def test_pdf_edit_agent_composes_edit_text_with_other_operations(runtime:
PdfEditPlanSelection(
operations=[ToolEndpoint.EDIT_TEXT, ToolEndpoint.ROTATE_PDF],
summary="Remove DRAFT marker, then rotate.",
rationale="test rationale",
),
parameter_selector=parameter_selector,
)
+14 -15
View File
@@ -9,7 +9,6 @@ from stirling.contracts import (
AiFile,
ExtractedFileText,
NeedIngestResponse,
PageText,
PdfContentType,
PdfQuestionAnswerResponse,
PdfQuestionNotFoundResponse,
@@ -18,8 +17,8 @@ from stirling.contracts import (
PdfTextSelection,
SupportedCapability,
)
from stirling.documents import Document, DocumentService, SqliteVecStore
from stirling.models import FileId
from stirling.rag import Document, RagService, SqliteVecStore
from stirling.services.runtime import AppRuntime
@@ -42,7 +41,7 @@ class StubEmbedder:
source: str = "",
base_metadata: dict[str, str] | None = None,
) -> list[Document]:
from stirling.documents.chunker import chunk_text
from stirling.rag.chunker import chunk_text
chunks = chunk_text(text, 100, 10)
docs: list[Document] = []
@@ -66,13 +65,13 @@ class StubPdfQuestionAgent(PdfQuestionAgent):
@pytest.fixture
def runtime_with_stub_rag(runtime: AppRuntime) -> AppRuntime:
"""A runtime whose document service uses a stub embedder + ephemeral store."""
stub = DocumentService(
"""A runtime whose RAG service uses a stub embedder + ephemeral store."""
stub = RagService(
embedder=StubEmbedder(), # type: ignore[arg-type]
store=SqliteVecStore.ephemeral(),
default_top_k=runtime.settings.rag_default_top_k,
)
return replace(runtime, documents=stub)
return replace(runtime, rag_service=stub)
@pytest.mark.anyio
@@ -90,9 +89,9 @@ async def test_requests_ingest_when_file_missing_from_rag(runtime_with_stub_rag:
@pytest.mark.anyio
async def test_reports_only_missing_files(runtime_with_stub_rag: AppRuntime) -> None:
await runtime_with_stub_rag.documents.ingest(
FileId("present-id"),
[PageText(page_number=1, text="Invoice total: 120.00.")],
await runtime_with_stub_rag.rag_service.index_text(
collection=FileId("present-id"),
text="Invoice total: 120.00.",
source="present.pdf",
)
agent = PdfQuestionAgent(runtime_with_stub_rag)
@@ -107,9 +106,9 @@ async def test_reports_only_missing_files(runtime_with_stub_rag: AppRuntime) ->
@pytest.mark.anyio
async def test_returns_grounded_answer_when_all_files_ingested(runtime_with_stub_rag: AppRuntime) -> None:
await runtime_with_stub_rag.documents.ingest(
FileId("invoice-id"),
[PageText(page_number=1, text="Invoice total: 120.00.")],
await runtime_with_stub_rag.rag_service.index_text(
collection=FileId("invoice-id"),
text="Invoice total: 120.00.",
source="invoice.pdf",
)
agent = StubPdfQuestionAgent(
@@ -138,9 +137,9 @@ async def test_returns_grounded_answer_when_all_files_ingested(runtime_with_stub
@pytest.mark.anyio
async def test_returns_not_found_when_answer_not_in_doc(runtime_with_stub_rag: AppRuntime) -> None:
await runtime_with_stub_rag.documents.ingest(
FileId("shipping-id"),
[PageText(page_number=1, text="This page contains only a shipping address.")],
await runtime_with_stub_rag.rag_service.index_text(
collection=FileId("shipping-id"),
text="This page contains only a shipping address.",
source="shipping.pdf",
)
agent = StubPdfQuestionAgent(
-138
View File
@@ -1,138 +0,0 @@
"""Tests for PDF to Markdown agent.
Two cases:
1. Narrative-only page: request validates and routes to reconstruction.
2. Mixed text + table page: layout with table region validates correctly.
"""
from __future__ import annotations
from stirling.contracts.pdf_to_markdown import (
LayoutFragment,
LayoutLine,
PageLayout,
PageLayoutArtifact,
PdfToMarkdownRequest,
PdfToMarkdownSuccessResponse,
)
def _frag(text: str, x: float, y: float, font_size: float = 10.0, bold: bool = False) -> LayoutFragment:
return LayoutFragment(text=text, x=x, y=y, width=float(len(text) * 6), font_size=font_size, bold=bold)
def _line(y: float, *frags: LayoutFragment) -> LayoutLine:
return LayoutLine(y=y, fragments=list(frags))
# ── Test 1: Narrative-only reconstruction ────────────────────────────────────────────────────────
# ── Contract test: Java serialization ↔ Python deserialization ──────────────────────────────────
# This JSON is also asserted field-by-field in PageLayoutArtifactContractTest.java.
# If either side renames a field, one of these tests fails.
_CONTRACT_JSON = (
'{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":'
'[{"pageNumber":1,"lines":[{"y":10.0,"fragments":'
'[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}'
)
def test_page_layout_artifact_deserialises_java_json() -> None:
artifact = PageLayoutArtifact.model_validate_json(_CONTRACT_JSON)
assert artifact.kind == "page_layout"
assert artifact.files[0].file_name == "test.pdf"
page = artifact.files[0].pages[0]
assert page.page_number == 1
line = page.lines[0]
assert line.y == 10.0
frag = line.fragments[0]
assert frag.text == "Hello"
assert frag.x == 1.0
assert frag.y == 2.0
assert frag.width == 30.0
assert frag.font_size == 12.0
assert frag.bold is True
def test_narrative_reconstruction_request_validates() -> None:
"""A prose-only page with no tables produces a valid PdfToMarkdownRequest."""
layout = PageLayout(
page_number=1,
lines=[
_line(72.0, _frag("Annual Report 2023", x=72.0, y=72.0, font_size=18.0, bold=True)),
_line(100.0, _frag("Our revenue grew significantly", x=72.0, y=100.0)),
_line(114.0, _frag("during the fiscal year ended", x=72.0, y=114.0)),
_line(128.0, _frag("December 31, 2023.", x=72.0, y=128.0)),
],
)
request = PdfToMarkdownRequest(
user_message="reconstruct this document",
page_layout=[layout],
)
assert len(request.page_layout) == 1
assert len(request.page_layout[0].lines) == 4
assert request.page_layout[0].lines[0].fragments[0].bold is True
assert request.page_layout[0].lines[0].fragments[0].font_size == 18.0
def test_narrative_reconstruction_response_validates() -> None:
"""PdfToMarkdownSuccessResponse accepts markdown and returns document_reconstructed outcome."""
response = PdfToMarkdownSuccessResponse(
markdown="# Annual Report 2023\n\nOur revenue grew significantly during the fiscal year.",
)
assert response.outcome == "document_reconstructed"
assert response.markdown.startswith("#")
# ── Test 2: Mixed text + table reconstruction ─────────────────────────────────────────────────────
def test_mixed_page_layout_validates() -> None:
"""A page with both prose lines and a table region produces a valid request."""
layout = PageLayout(
page_number=1,
lines=[
# Prose heading
_line(50.0, _frag("Projects in Development", x=72.0, y=50.0, font_size=14.0, bold=True)),
# Table header row
_line(
80.0,
_frag("Project Name", x=72.0, y=80.0, bold=True),
_frag("Location", x=200.0, y=80.0, bold=True),
_frag("Size (MW)", x=290.0, y=80.0, bold=True),
),
# Table data rows
_line(
95.0,
_frag("Chaplin Wind 1", x=72.0, y=95.0),
_frag("Saskatchewan", x=200.0, y=95.0),
_frag("177", x=290.0, y=95.0),
),
_line(
110.0,
_frag("Amherst Island 2", x=72.0, y=110.0),
_frag("Ontario", x=200.0, y=110.0),
_frag("75", x=290.0, y=110.0),
),
# Prose after table
_line(140.0, _frag("Notes:", x=72.0, y=140.0, bold=True)),
_line(154.0, _frag("1 PPA signed", x=85.0, y=154.0)),
],
)
request = PdfToMarkdownRequest(
user_message="markdown",
page_layout=[layout],
)
assert len(request.page_layout[0].lines) == 6
# Header line has 3 fragments at distinct x-positions (column detection)
header_line = request.page_layout[0].lines[1]
xs = [f.x for f in header_line.fragments]
assert xs == [72.0, 200.0, 290.0]
# Data rows have matching x-positions
data_row = request.page_layout[0].lines[2]
assert [f.x for f in data_row.fragments] == [72.0, 200.0, 290.0]
@@ -2,15 +2,14 @@ from __future__ import annotations
import pytest
from stirling.contracts import PageText
from stirling.documents.chunker import chunk_text
from stirling.documents.rag_capability import RagCapability
from stirling.documents.service import DocumentService
from stirling.documents.sqlite_vec_store import SqliteVecStore
from stirling.documents.store import Document, SearchResult
from stirling.models import FileId
from stirling.rag.capability import RagCapability
from stirling.rag.chunker import chunk_text
from stirling.rag.service import RagService
from stirling.rag.sqlite_vec_store import SqliteVecStore
from stirling.rag.store import Document, SearchResult
# chunk_text
# ── chunk_text ──────────────────────────────────────────────────────────
class TestChunkText:
@@ -27,6 +26,7 @@ class TestChunkText:
def test_splits_on_paragraph_boundaries(self) -> None:
text = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph."
chunks = chunk_text(text, chunk_size=30, overlap=0)
# Each paragraph fits in 30 chars, so they should be split
assert len(chunks) >= 2
assert "First paragraph." in chunks[0]
@@ -35,19 +35,22 @@ class TestChunkText:
chunks = chunk_text(text, chunk_size=100, overlap=10)
assert len(chunks) > 1
for chunk in chunks:
assert len(chunk) <= 200
# Chunks may slightly exceed due to sentence boundary snapping
assert len(chunk) <= 200 # generous upper bound
def test_overlap_produces_shared_content(self) -> None:
sentences = [f"Sentence number {i}." for i in range(20)]
text = " ".join(sentences)
chunks = chunk_text(text, chunk_size=100, overlap=30)
if len(chunks) >= 2:
words_in_first_tail = chunks[0].split()[-3:]
# After word-boundary snapping, the second chunk should share
# some content with the tail of the first chunk
words_in_first_tail = chunks[0].split()[-3:] # last 3 words
overlap_text = " ".join(words_in_first_tail)
assert overlap_text in chunks[1], f"Expected overlap '{overlap_text}' in chunk[1]: '{chunks[1][:80]}...'"
# SqliteVecStore
# ── SqliteVecStore ──────────────────────────────────────────────────────
class TestSqliteVecStore:
@@ -56,12 +59,12 @@ class TestSqliteVecStore:
@pytest.mark.anyio
async def test_add_and_search(self) -> None:
store = SqliteVecStore.ephemeral()
await store.ensure_collection("test-col", "test.pdf")
docs = [
Document(id="1", text="Python is a programming language", metadata={"source": "test"}),
Document(id="2", text="Java is another programming language", metadata={"source": "test"}),
Document(id="3", text="The weather today is sunny", metadata={"source": "test"}),
]
# Simple 3-dimensional embeddings for testing
embeddings = [
[1.0, 0.0, 0.0],
[0.9, 0.1, 0.0],
@@ -69,16 +72,17 @@ class TestSqliteVecStore:
]
await store.add_documents("test-col", docs, embeddings)
# Search with a query close to the programming-related docs
results = await store.search("test-col", [1.0, 0.05, 0.0], top_k=2)
assert len(results) == 2
assert isinstance(results[0], SearchResult)
# The closest should be doc "1" (exact match on first dimension)
assert results[0].document.id == "1"
assert results[0].score > 0.5
@pytest.mark.anyio
async def test_list_and_has_collection(self) -> None:
store = SqliteVecStore.ephemeral()
await store.ensure_collection("my-collection", "test.pdf")
docs = [Document(id="1", text="test", metadata={})]
await store.add_documents("my-collection", docs, [[1.0, 0.0]])
@@ -90,7 +94,6 @@ class TestSqliteVecStore:
@pytest.mark.anyio
async def test_delete_collection(self) -> None:
store = SqliteVecStore.ephemeral()
await store.ensure_collection("to-delete", "test.pdf")
docs = [Document(id="1", text="test", metadata={})]
await store.add_documents("to-delete", docs, [[1.0]])
@@ -101,7 +104,6 @@ class TestSqliteVecStore:
@pytest.mark.anyio
async def test_search_empty_collection(self) -> None:
store = SqliteVecStore.ephemeral()
await store.ensure_collection("empty-test", "test.pdf")
docs = [Document(id="1", text="test", metadata={})]
await store.add_documents("empty-test", docs, [[1.0, 0.0]])
results = await store.search("empty-test", [1.0, 0.0], top_k=5)
@@ -115,7 +117,7 @@ class TestSqliteVecStore:
await store.add_documents("bad", docs, [[1.0], [2.0]])
# DocumentService (with stub embedder)
# ── RagService (with stub embedder) ────────────────────────────────────
class StubEmbeddingService:
@@ -125,6 +127,7 @@ class StubEmbeddingService:
self._dim = dim
async def embed_query(self, text: str) -> list[float]:
# Deterministic embedding based on hash of text
h = hash(text) % 1000
return [(h + i) / 1000.0 for i in range(self._dim)]
@@ -137,6 +140,8 @@ class StubEmbeddingService:
source: str = "",
base_metadata: dict[str, str] | None = None,
) -> list[Document]:
from stirling.rag.chunker import chunk_text
chunks = chunk_text(text, 100, 10)
docs = []
for i, chunk in enumerate(chunks):
@@ -149,113 +154,53 @@ class StubEmbeddingService:
@pytest.fixture
def documents() -> DocumentService:
"""Each DocumentService test gets its own fresh ephemeral store to avoid dimension conflicts."""
def rag_service() -> RagService:
"""Each RagService test gets its own fresh ephemeral store to avoid dimension conflicts."""
store = SqliteVecStore.ephemeral()
return DocumentService(embedder=StubEmbeddingService(), store=store, default_top_k=3) # type: ignore[arg-type]
return RagService(embedder=StubEmbeddingService(), store=store, default_top_k=3) # type: ignore[arg-type]
def _pages(text: str) -> list[PageText]:
return [PageText(page_number=1, text=text)]
class TestDocumentService:
class TestRagService:
@pytest.mark.anyio
async def test_ingest_and_search(self, documents: DocumentService) -> None:
async def test_index_and_search(self, rag_service: RagService) -> None:
text = "Python is great for data science. It has many libraries like pandas and numpy."
count = await documents.ingest(FileId("docs"), _pages(text), source="guide.pdf")
count = await rag_service.index_text(FileId("docs"), text, source="guide.pdf")
assert count > 0
results = await documents.search("Python libraries", collection=FileId("docs"))
results = await rag_service.search("Python libraries", collection=FileId("docs"))
assert len(results) > 0
assert results[0].document.text
assert results[0].document.text # non-empty text
@pytest.mark.anyio
async def test_ingest_empty_text_returns_zero_chunks(self, documents: DocumentService) -> None:
count = await documents.ingest(FileId("docs"), _pages(""), source="empty.pdf")
async def test_index_empty_text_returns_zero(self, rag_service: RagService) -> None:
count = await rag_service.index_text(FileId("docs"), "", source="empty.pdf")
assert count == 0
@pytest.mark.anyio
async def test_search_nonexistent_collection_returns_empty(self, documents: DocumentService) -> None:
results = await documents.search("anything", collection=FileId("nonexistent"))
async def test_search_nonexistent_collection_returns_empty(self, rag_service: RagService) -> None:
results = await rag_service.search("anything", collection=FileId("nonexistent"))
assert results == []
@pytest.mark.anyio
async def test_search_all_collections(self, documents: DocumentService) -> None:
await documents.ingest(FileId("col-a"), _pages("Machine learning overview."), source="ml.pdf")
await documents.ingest(FileId("col-b"), _pages("Deep learning with neural networks."), source="dl.pdf")
async def test_search_all_collections(self, rag_service: RagService) -> None:
await rag_service.index_text(FileId("col-a"), "Machine learning overview.", source="ml.pdf")
await rag_service.index_text(FileId("col-b"), "Deep learning with neural networks.", source="dl.pdf")
results = await documents.search("neural networks")
results = await rag_service.search("neural networks")
assert len(results) > 0
@pytest.mark.anyio
async def test_delete_collection(self, documents: DocumentService) -> None:
await documents.ingest(FileId("temp"), _pages("Temporary data."), source="tmp.pdf")
collections = await documents.list_collections()
async def test_delete_collection(self, rag_service: RagService) -> None:
await rag_service.index_text(FileId("temp"), "Temporary data.", source="tmp.pdf")
collections = await rag_service.list_collections()
assert "temp" in collections
await documents.delete_collection(FileId("temp"))
collections = await documents.list_collections()
await rag_service.delete_collection(FileId("temp"))
collections = await rag_service.list_collections()
assert "temp" not in collections
@pytest.mark.anyio
async def test_ingest_stores_pages_in_order(self, documents: DocumentService) -> None:
pages = [
PageText(page_number=1, text="First page text."),
PageText(page_number=2, text="Second page text."),
PageText(page_number=3, text="Third page text."),
]
await documents.ingest(FileId("ordered"), pages, source="ordered.pdf")
stored = await documents.read_pages(FileId("ordered"))
assert [p.page_number for p in stored] == [1, 2, 3]
assert stored[0].text == "First page text."
assert stored[0].char_count == len("First page text.")
@pytest.mark.anyio
async def test_read_pages_with_range(self, documents: DocumentService) -> None:
from stirling.contracts import PageRange
pages = [PageText(page_number=i, text=f"page {i}") for i in range(1, 6)]
await documents.ingest(FileId("ranged"), pages, source="r.pdf")
subset = await documents.read_pages(FileId("ranged"), PageRange(start=2, end=4))
assert [p.page_number for p in subset] == [2, 3, 4]
@pytest.mark.anyio
async def test_ingest_replaces_previous_pages(self, documents: DocumentService) -> None:
await documents.ingest(
FileId("doc"),
[PageText(page_number=1, text="old"), PageText(page_number=2, text="old2")],
source="v1.pdf",
)
await documents.ingest(
FileId("doc"),
[PageText(page_number=1, text="new")],
source="v2.pdf",
)
stored = await documents.read_pages(FileId("doc"))
assert [p.page_number for p in stored] == [1]
assert stored[0].text == "new"
@pytest.mark.anyio
async def test_ingest_keeps_blank_pages_in_page_store(self, documents: DocumentService) -> None:
"""Blank pages are skipped for embedding but retained in the page store
so page numbering stays continuous when reading back."""
pages = [
PageText(page_number=1, text="Real text on page 1."),
PageText(page_number=2, text=" "),
PageText(page_number=3, text="Real text on page 3."),
]
await documents.ingest(FileId("with-blanks"), pages, source="blanks.pdf")
stored = await documents.read_pages(FileId("with-blanks"))
assert [p.page_number for p in stored] == [1, 2, 3]
assert stored[1].text.strip() == ""
# RagCapability
# ── RagCapability ──────────────────────────────────────────────────────
async def _invoke_search_knowledge(capability: RagCapability, query: str, max_results: int = 5) -> str:
@@ -265,27 +210,27 @@ async def _invoke_search_knowledge(capability: RagCapability, query: str, max_re
toolset = capability.toolset
assert isinstance(toolset, FunctionToolset)
tool = toolset.tools["search_knowledge"]
return await tool.function(query=query, max_results=max_results) # type: ignore[call-arg]
return await tool.function(query=query, max_results=max_results) # type: ignore[call-arg] — pyright can't infer the generic tool function's kwargs
class TestRagCapability:
def test_instructions_static_when_collections_pinned(self, documents: DocumentService) -> None:
cap = RagCapability(documents, collections=[FileId("docs"), FileId("manuals")])
def test_instructions_static_when_collections_pinned(self, rag_service: RagService) -> None:
cap = RagCapability(rag_service, collections=[FileId("docs"), FileId("manuals")])
instructions = cap.instructions
assert isinstance(instructions, str)
assert "docs, manuals" in instructions
assert "search_knowledge" in instructions
def test_instructions_dynamic_when_no_collections(self, documents: DocumentService) -> None:
cap = RagCapability(documents)
def test_instructions_dynamic_when_no_collections(self, rag_service: RagService) -> None:
cap = RagCapability(rag_service)
instructions = cap.instructions
assert callable(instructions)
@pytest.mark.anyio
async def test_dynamic_instructions_list_available_collections(self, documents: DocumentService) -> None:
await documents.ingest(FileId("col-a"), _pages("Alpha content."), source="a.pdf")
await documents.ingest(FileId("col-b"), _pages("Beta content."), source="b.pdf")
cap = RagCapability(documents)
async def test_dynamic_instructions_list_available_collections(self, rag_service: RagService) -> None:
await rag_service.index_text(FileId("col-a"), "Alpha content.", source="a.pdf")
await rag_service.index_text(FileId("col-b"), "Beta content.", source="b.pdf")
cap = RagCapability(rag_service)
instructions_fn = cap.instructions
assert callable(instructions_fn)
text = await instructions_fn()
@@ -293,23 +238,23 @@ class TestRagCapability:
assert "col-b" in text
@pytest.mark.anyio
async def test_dynamic_instructions_when_store_empty(self, documents: DocumentService) -> None:
cap = RagCapability(documents)
async def test_dynamic_instructions_when_store_empty(self, rag_service: RagService) -> None:
cap = RagCapability(rag_service)
instructions_fn = cap.instructions
assert callable(instructions_fn)
text = await instructions_fn()
assert "empty" in text.lower()
@pytest.mark.anyio
async def test_search_knowledge_returns_no_results_message_when_empty(self, documents: DocumentService) -> None:
cap = RagCapability(documents)
async def test_search_knowledge_returns_no_results_message_when_empty(self, rag_service: RagService) -> None:
cap = RagCapability(rag_service)
output = await _invoke_search_knowledge(cap, "anything")
assert output == "No relevant results found in the knowledge base."
@pytest.mark.anyio
async def test_search_knowledge_formats_results_with_source_and_score(self, documents: DocumentService) -> None:
await documents.ingest(FileId("docs"), _pages("Python is a programming language."), source="guide.pdf")
cap = RagCapability(documents)
async def test_search_knowledge_formats_results_with_source_and_score(self, rag_service: RagService) -> None:
await rag_service.index_text(FileId("docs"), "Python is a programming language.", source="guide.pdf")
cap = RagCapability(rag_service)
output = await _invoke_search_knowledge(cap, "Python")
assert "[Result 1" in output
assert "source: guide.pdf" in output
@@ -317,39 +262,43 @@ class TestRagCapability:
assert "relevance:" in output
@pytest.mark.anyio
async def test_search_knowledge_restricts_to_pinned_collections(self, documents: DocumentService) -> None:
await documents.ingest(FileId("pinned"), _pages("Pinned collection content."), source="pinned.pdf")
await documents.ingest(FileId("other"), _pages("Content in another collection."), source="other.pdf")
async def test_search_knowledge_restricts_to_pinned_collections(self, rag_service: RagService) -> None:
await rag_service.index_text(FileId("pinned"), "Pinned collection content.", source="pinned.pdf")
await rag_service.index_text(FileId("other"), "Content in another collection.", source="other.pdf")
cap = RagCapability(documents, collections=[FileId("pinned")])
cap = RagCapability(rag_service, collections=[FileId("pinned")])
output = await _invoke_search_knowledge(cap, "content")
assert "pinned.pdf" in output
assert "other.pdf" not in output
@pytest.mark.anyio
async def test_search_knowledge_respects_max_results(self, documents: DocumentService) -> None:
async def test_search_knowledge_respects_max_results(self, rag_service: RagService) -> None:
paragraphs = "\n\n".join(f"Paragraph {i} about topic." for i in range(10))
await documents.ingest(FileId("bulk"), _pages(paragraphs), source="bulk.pdf")
await rag_service.index_text(FileId("bulk"), paragraphs, source="bulk.pdf")
cap = RagCapability(documents)
cap = RagCapability(rag_service)
output = await _invoke_search_knowledge(cap, "topic", max_results=2)
# Only two results requested, so only Result 1 and Result 2 should appear
assert "[Result 1" in output
assert "[Result 2" in output
assert "[Result 3" not in output
@pytest.mark.anyio
async def test_search_knowledge_tool_is_hidden_after_budget_exhausted(self, documents: DocumentService) -> None:
async def test_search_knowledge_tool_is_hidden_after_budget_exhausted(self, rag_service: RagService) -> None:
"""The prepare callback must return None once max_searches has been reached
so the agent can no longer call the tool on subsequent turns."""
await documents.ingest(FileId("docs"), _pages("Some content."), source="x.pdf")
cap = RagCapability(documents, max_searches=2)
await rag_service.index_text(FileId("docs"), "Some content.", source="x.pdf")
cap = RagCapability(rag_service, max_searches=2)
tool_def = _dummy_tool_def()
# Budget intact: prepare returns the tool definition.
assert await cap._prepare_search_knowledge(None, tool_def) is tool_def # type: ignore[arg-type]
# Use the budget.
await _invoke_search_knowledge(cap, "content")
await _invoke_search_knowledge(cap, "content")
# Budget spent: prepare returns None, removing the tool from the agent's next turn.
assert await cap._prepare_search_knowledge(None, tool_def) is None # type: ignore[arg-type]
@@ -6,9 +6,9 @@ import pytest
from fastapi.testclient import TestClient
from stirling.api import app
from stirling.api.dependencies import get_document_service
from stirling.documents import Document, DocumentService, SqliteVecStore
from stirling.api.dependencies import get_rag_service
from stirling.models import FileId
from stirling.rag import Document, RagService, SqliteVecStore
class StubEmbedder:
@@ -30,7 +30,7 @@ class StubEmbedder:
source: str = "",
base_metadata: dict[str, str] | None = None,
) -> list[Document]:
from stirling.documents.chunker import chunk_text
from stirling.rag.chunker import chunk_text
chunks = chunk_text(text, 100, 10)
docs = []
@@ -43,8 +43,8 @@ class StubEmbedder:
return docs
def _build_service() -> DocumentService:
return DocumentService(
def _build_service() -> RagService:
return RagService(
embedder=StubEmbedder(), # type: ignore[arg-type]
store=SqliteVecStore.ephemeral(),
default_top_k=3,
@@ -52,25 +52,25 @@ def _build_service() -> DocumentService:
@pytest.fixture
def service() -> DocumentService:
def service() -> RagService:
return _build_service()
@pytest.fixture
def client(service: DocumentService) -> Iterator[TestClient]:
app.dependency_overrides[get_document_service] = lambda: service
def client(service: RagService) -> Iterator[TestClient]:
app.dependency_overrides[get_rag_service] = lambda: service
try:
yield TestClient(app)
finally:
app.dependency_overrides.pop(get_document_service, None)
app.dependency_overrides.pop(get_rag_service, None)
# ── POST /documents ─────────────────────────────────────────────────────
def test_ingest_document_indexes_page_text(client: TestClient, service: DocumentService) -> None:
def test_ingest_document_indexes_page_text(client: TestClient, service: RagService) -> None:
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={
"documentId": "doc-123",
"source": "report.pdf",
@@ -87,9 +87,9 @@ def test_ingest_document_indexes_page_text(client: TestClient, service: Document
@pytest.mark.anyio
async def test_ingest_document_replaces_existing_content(client: TestClient, service: DocumentService) -> None:
async def test_ingest_document_replaces_existing_content(client: TestClient, service: RagService) -> None:
client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={
"documentId": "replace-me",
"source": "replace-me.pdf",
@@ -98,7 +98,7 @@ async def test_ingest_document_replaces_existing_content(client: TestClient, ser
)
# Second ingest with different content should replace the first entirely
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={
"documentId": "replace-me",
"source": "replace-me.pdf",
@@ -115,7 +115,7 @@ async def test_ingest_document_replaces_existing_content(client: TestClient, ser
def test_ingest_document_skips_empty_pages(client: TestClient) -> None:
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={
"documentId": "mixed",
"source": "mixed.pdf",
@@ -130,14 +130,14 @@ def test_ingest_document_skips_empty_pages(client: TestClient) -> None:
def test_ingest_document_with_no_content_returns_zero(client: TestClient) -> None:
response = client.post("/api/v1/documents", json={"documentId": "empty", "source": "empty.pdf"})
response = client.post("/api/v1/rag/documents", json={"documentId": "empty", "source": "empty.pdf"})
assert response.status_code == 200
assert response.json()["chunksIndexed"] == 0
def test_ingest_document_rejects_empty_id(client: TestClient) -> None:
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={"documentId": "", "source": "x.pdf", "pageText": [{"pageNumber": 1, "text": "something"}]},
)
assert response.status_code == 422
@@ -145,7 +145,7 @@ def test_ingest_document_rejects_empty_id(client: TestClient) -> None:
def test_ingest_document_rejects_missing_source(client: TestClient) -> None:
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={"documentId": "doc-1", "pageText": [{"pageNumber": 1, "text": "something"}]},
)
assert response.status_code == 422
@@ -153,7 +153,7 @@ def test_ingest_document_rejects_missing_source(client: TestClient) -> None:
def test_ingest_document_rejects_empty_source(client: TestClient) -> None:
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={"documentId": "doc-1", "source": "", "pageText": [{"pageNumber": 1, "text": "something"}]},
)
assert response.status_code == 422
@@ -161,7 +161,7 @@ def test_ingest_document_rejects_empty_source(client: TestClient) -> None:
def test_ingest_document_rejects_non_positive_page_number(client: TestClient) -> None:
response = client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={
"documentId": "bad-page",
"source": "bad-page.pdf",
@@ -176,30 +176,30 @@ def test_ingest_document_rejects_non_positive_page_number(client: TestClient) ->
def test_delete_document_reports_deleted_true_when_existed(client: TestClient) -> None:
client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={
"documentId": "to-delete",
"source": "to-delete.pdf",
"pageText": [{"pageNumber": 1, "text": "Text."}],
},
)
response = client.delete("/api/v1/documents/to-delete")
response = client.delete("/api/v1/rag/documents/to-delete")
assert response.status_code == 200
assert response.json() == {"documentId": "to-delete", "deleted": True}
def test_delete_document_is_idempotent(client: TestClient) -> None:
response = client.delete("/api/v1/documents/never-existed")
response = client.delete("/api/v1/rag/documents/never-existed")
assert response.status_code == 200
assert response.json() == {"documentId": "never-existed", "deleted": False}
@pytest.mark.anyio
async def test_delete_document_removes_collection(client: TestClient, service: DocumentService) -> None:
async def test_delete_document_removes_collection(client: TestClient, service: RagService) -> None:
client.post(
"/api/v1/documents",
"/api/v1/rag/documents",
json={"documentId": "gone", "source": "gone.pdf", "pageText": [{"pageNumber": 1, "text": "Text."}]},
)
assert await service.has_collection(FileId("gone"))
client.delete("/api/v1/documents/gone")
client.delete("/api/v1/rag/documents/gone")
assert not await service.has_collection(FileId("gone"))
+5 -115
View File
@@ -1,7 +1,3 @@
import asyncio
import json
from unittest.mock import patch
from conftest import build_app_settings
from fastapi.testclient import TestClient
@@ -29,11 +25,8 @@ from stirling.contracts import (
PdfQuestionNotFoundResponse,
PdfQuestionRequest,
SupportedCapability,
WholeDocReadStarted,
WholeDocSliceDone,
)
from stirling.models.tool_models import Angle, RotatePdfParams
from stirling.services import emit_progress
class StubOrchestratorAgent:
@@ -47,34 +40,6 @@ class StubOrchestratorAgent:
)
class StubProgressOrchestratorAgent:
"""Orchestrator stub that emits two progress events before returning.
Used to verify the streaming endpoint plumbs the ContextVar emitter through
to deep callees and forwards events as NDJSON in order.
"""
async def handle(self, request: OrchestratorRequest) -> NeedContentResponse:
await emit_progress(WholeDocReadStarted(question="x", pages=10, slices=2))
await emit_progress(
WholeDocSliceDone(
completed=1,
total=2,
pages="pages=1-5",
duration_ms=42,
excerpts=2,
facts=3,
)
)
return NeedContentResponse(
resume_with=SupportedCapability.PDF_QUESTION,
reason=request.user_message,
files=[],
max_pages=1,
max_characters=1000,
)
class StubPdfEditAgent:
async def handle(self, request: PdfEditRequest) -> EditCannotDoResponse:
return EditCannotDoResponse(reason=request.user_message)
@@ -122,89 +87,14 @@ def test_health_route() -> None:
assert response.json()["status"] == "ok"
def test_orchestrator_route_streams_result_only_when_no_progress() -> None:
"""The orchestrator endpoint always streams NDJSON. An agent that emits no
progress events still produces a single ``result`` frame with the typed
response body."""
with client.stream(
"POST",
def test_orchestrator_route() -> None:
response = client.post(
"/api/v1/orchestrator",
json={"userMessage": "route this", "files": [{"id": "test-id", "name": "test.pdf"}]},
) as response:
assert response.status_code == 200
events = [json.loads(line) for line in response.iter_lines() if line]
)
assert [e["event"] for e in events] == ["result"]
body = events[0]["response"]
assert body["outcome"] == "need_content"
def test_orchestrator_route_streams_progress_then_result() -> None:
"""When an agent emits progress via the ContextVar emitter, those frames
arrive on the wire before the final result frame."""
app.dependency_overrides[get_orchestrator_agent] = lambda: StubProgressOrchestratorAgent()
try:
with client.stream(
"POST",
"/api/v1/orchestrator",
json={"userMessage": "stream this", "files": [{"id": "test-id", "name": "test.pdf"}]},
) as response:
assert response.status_code == 200
events = [json.loads(line) for line in response.iter_lines() if line]
finally:
app.dependency_overrides[get_orchestrator_agent] = lambda: StubOrchestratorAgent()
progress = [e for e in events if e["event"] == "progress"]
results = [e for e in events if e["event"] == "result"]
assert [p["phase"] for p in progress] == ["whole_doc_read_started", "whole_doc_slice_done"]
assert progress[1]["completed"] == 1
assert len(results) == 1
response = results[0]["response"]
assert response["outcome"] == "need_content"
# Wire format must be camelCase: Java's Jackson deserializer expects camelCase
# field names. ``maxPages`` here doubles as a regression guard against the
# snake_case bug that surfaced as "need_ingest without listing any files to ingest".
assert "maxPages" in response
assert "max_pages" not in response
def test_orchestrator_route_emits_heartbeats_while_agent_is_busy() -> None:
"""While the agent is in flight, the streaming endpoint emits heartbeat
frames at the configured cadence so each layer of the connection stays
visibly alive and disconnects propagate within bounded latency."""
class _SlowAgent:
async def handle(self, _request: OrchestratorRequest) -> NeedContentResponse:
# Sleep long enough for several heartbeats at the patched cadence.
await asyncio.sleep(0.2)
return NeedContentResponse(
resume_with=SupportedCapability.PDF_QUESTION,
reason="ok",
files=[],
max_pages=1,
max_characters=1000,
)
app.dependency_overrides[get_orchestrator_agent] = lambda: _SlowAgent()
try:
with patch("stirling.api.routes.orchestrator.HEARTBEAT_INTERVAL_SECONDS", 0.03):
with client.stream(
"POST",
"/api/v1/orchestrator",
json={"userMessage": "wait", "files": [{"id": "test-id", "name": "test.pdf"}]},
) as response:
assert response.status_code == 200
events = [json.loads(line) for line in response.iter_lines() if line]
finally:
app.dependency_overrides[get_orchestrator_agent] = lambda: StubOrchestratorAgent()
heartbeats = [e for e in events if e["event"] == "heartbeat"]
results = [e for e in events if e["event"] == "result"]
# At least a couple of heartbeats fired during the 0.2s agent sleep at 0.03s cadence.
assert len(heartbeats) >= 2
# The result still arrives after the agent finishes.
assert len(results) == 1
assert results[0]["response"]["outcome"] == "need_content"
assert response.status_code == 200
assert response.json()["outcome"] == "need_content"
def test_pdf_edit_route() -> None:
-4
View File
@@ -92,10 +92,6 @@ def test_app_settings_accepts_model_configuration() -> None:
rag_chunk_overlap=64,
rag_default_top_k=5,
rag_max_searches=5,
chunked_reasoner_chars_per_slice=16_000,
chunked_reasoner_concurrency=10,
chunked_reasoner_worker_timeout_seconds=60.0,
chunked_reasoner_notes_char_budget=250_000,
max_pages=200,
max_characters=200_000,
posthog_enabled=False,
-2
View File
@@ -2,8 +2,6 @@ dist/
# Tauri/Cargo build output (binary assets named *.js etc. confuse Prettier)
src-tauri/target/
node_modules/
src-tauri/target/
src-tauri/gen/
public/vendor/
public/pdfjs*/
public/js/thirdParty/
+35 -1468
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More