diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index ab2b605f62..30fb37e474 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -558,6 +558,39 @@ close = "Close" error = "Error" expand = "Expand" +[admin.formDetection] +active = "Active" +description = "Install the AI model used to auto-detect form fields. The selected model is downloaded on demand (about 40-100MB) into the configs volume and is not bundled with Stirling-PDF." +enableFeature = "Enable feature" +engineDescription = "Browser keeps the PDF on the device (downloads a ~12MB runtime once, then cached); Server runs it on the backend; Auto prefers the browser and falls back to the server." +engineLabel = "Where detection runs" +install = "Install" +license = "License" +notAvailable = "This catalog entry has no download URL/checksum configured yet, so it cannot be installed." +notWritable = "The model directory is not writable; check the configs volume mount." +selectModel = "Model" +selectPlaceholder = "Select a model" +status = "Status" +switch = "Switch to this model" +title = "AI Form Detection" +uninstall = "Uninstall" + +[admin.formDetection.airgap] +hide = "Hide offline install instructions" +intro = "No internet on the server? Install the model manually:" +noSha = "(checksum not set)" +show = "Air-gapped / offline install instructions" +step1 = "1. On a machine with internet, download the model file:" +step2 = "2. Verify its SHA-256 checksum matches:" +step3 = "3. Copy it onto the Stirling-PDF server into the model directory:" +step4 = "4. Set formDetection.activeModelId to this model's id in settings.yml, then restart (an installed model is auto-detected on boot). is the configs volume (e.g. /configs in Docker). Alternatively, point the Install button at an internal mirror using an override URL + checksum." + +[admin.formDetection.engine] +auto = "Auto" +browser = "Browser" +server = "Server" +serverUnavailable = "The server engine is not bundled in this build, so detection runs in the browser. Use Auto or Browser." + [admin.settings] discard = "Discard" error = "Failed to save settings" @@ -1837,6 +1870,16 @@ bullet2 = "Creates a clean, valid filename from the detected title" bullet3 = "Keeps the original name if no suitable title is found" title = "Smart Renaming" +[autoFormDetection] +loading = "Detecting form fields..." +submit = "Detect & make fillable" + +[autoFormDetection.error] +failed = "An error occurred while detecting form fields." + +[autoFormDetection.results] +title = "Detected Form Fields" + [automate] copyToSaved = "Copy to Saved" desc = "Build multi-step workflows by chaining together PDF actions. Ideal for recurring tasks." diff --git a/frontend/editor/src/core/hooks/tools/autoFormDetection/useAutoFormDetectionOperation.ts b/frontend/editor/src/core/hooks/tools/autoFormDetection/useAutoFormDetectionOperation.ts index c5bfe1fc48..88a84387c1 100644 --- a/frontend/editor/src/core/hooks/tools/autoFormDetection/useAutoFormDetectionOperation.ts +++ b/frontend/editor/src/core/hooks/tools/autoFormDetection/useAutoFormDetectionOperation.ts @@ -59,9 +59,8 @@ async function browserDetect( ): Promise { // Lazy-load the in-browser engine (onnxruntime-web + the ~12MB wasm) only when browser-mode // detection actually runs - it is never pulled into the initial bundle or loaded on the homepage. - const { runBrowserDetection } = await import( - "@app/services/formDetection/runBrowserPipeline" - ); + const { runBrowserDetection } = + await import("@app/services/formDetection/runBrowserPipeline"); const bytes = await file.arrayBuffer(); const { appliedPdf } = await runBrowserDetection( bytes, diff --git a/frontend/editor/src/core/services/formDetection/decode.test.ts b/frontend/editor/src/core/services/formDetection/decode.test.ts index 9190fe62f2..a05706a0ce 100644 --- a/frontend/editor/src/core/services/formDetection/decode.test.ts +++ b/frontend/editor/src/core/services/formDetection/decode.test.ts @@ -43,12 +43,24 @@ const pre: Preprocessed = { // nc_first layout [channels=6][anchors=3], data[c*anchors + a] // box A (cx5,cy5,w4,h4) twice (overlapping) + box B (cx8,cy8,w2,h2) const data = [ - 5, 5, 8, // cx - 5, 5, 8, // cy - 4, 4, 2, // w - 4, 4, 2, // h - 0.9, 0.8, 0.7, // text score - 0.1, 0.1, 0.1, // choice score + 5, + 5, + 8, // cx + 5, + 5, + 8, // cy + 4, + 4, + 2, // w + 4, + 4, + 2, // h + 0.9, + 0.8, + 0.7, // text score + 0.1, + 0.1, + 0.1, // choice score ]; const out: RawOutput = { data, d1: 6, d2: 3 }; diff --git a/frontend/editor/src/core/services/formDetection/decode.ts b/frontend/editor/src/core/services/formDetection/decode.ts index eba56e6919..3dd32b76f4 100644 --- a/frontend/editor/src/core/services/formDetection/decode.ts +++ b/frontend/editor/src/core/services/formDetection/decode.ts @@ -1,7 +1,12 @@ // Pure decode/NMS/un-projection - a 1:1 port of Yolo.decode in the backend. Kept free of any // browser API so it can be unit-tested for parity against the Java golden output. -import { Detection, ModelPipelineSpec, Preprocessed, RawOutput } from "@app/services/formDetection/types"; +import { + Detection, + ModelPipelineSpec, + Preprocessed, + RawOutput, +} from "@app/services/formDetection/types"; function at( data: Float32Array | number[], @@ -30,7 +35,11 @@ function iou(a: Detection, b: Detection): number { return union <= 0 ? 0 : inter / union; } -function nms(dets: Detection[], mode: string, iouThreshold: number): Detection[] { +function nms( + dets: Detection[], + mode: string, + iouThreshold: number, +): Detection[] { if (dets.length < 2 || (mode ?? "").toLowerCase() === "none") { return dets; } @@ -78,7 +87,8 @@ export function decode( let bestClass = -1; let bestScore = 0; for (let c = 0; c < numClasses; c++) { - const s = at(data, ncFirst, anchors, channels, classOffset + c, a) * objScore; + const s = + at(data, ncFirst, anchors, channels, classOffset + c, a) * objScore; if (s > bestScore) { bestScore = s; bestClass = c; @@ -100,7 +110,14 @@ export function decode( ow = Math.max(0, Math.min(ow, pre.srcW - cxl)); oh = Math.max(0, Math.min(oh, pre.srcH - cyl)); if (ow <= 0 || oh <= 0) continue; - dets.push({ classId: bestClass, score: bestScore, x: cxl, y: cyl, w: ow, h: oh }); + dets.push({ + classId: bestClass, + score: bestScore, + x: cxl, + y: cyl, + w: ow, + h: oh, + }); } return nms(dets, spec.nms, spec.iou); } diff --git a/frontend/editor/src/core/services/formDetection/modelCache.ts b/frontend/editor/src/core/services/formDetection/modelCache.ts index 582bc5ce0f..98f2d37775 100644 --- a/frontend/editor/src/core/services/formDetection/modelCache.ts +++ b/frontend/editor/src/core/services/formDetection/modelCache.ts @@ -28,7 +28,9 @@ async function verify(bytes: ArrayBuffer, expectedSha?: string): Promise { * Return the active model bytes, from the Cache API when present (and checksum-valid) or by * downloading from the backend. The cache key is the checksum, so a model swap naturally misses. */ -export async function loadModelBytes(expectedSha?: string): Promise { +export async function loadModelBytes( + expectedSha?: string, +): Promise { const cacheKey = `${MODEL_FILE_URL}#${expectedSha ?? "nosha"}`; // Cache API is unavailable in non-secure contexts; degrade to a plain download in that case. const cache = await caches.open(CACHE_NAME).catch(() => null); diff --git a/frontend/editor/src/core/services/formDetection/pdfRender.ts b/frontend/editor/src/core/services/formDetection/pdfRender.ts index e65508e761..4545b9f3b6 100644 --- a/frontend/editor/src/core/services/formDetection/pdfRender.ts +++ b/frontend/editor/src/core/services/formDetection/pdfRender.ts @@ -46,7 +46,8 @@ export async function renderPages( const pdfjs = window.pdfjsLib; if (!pdfjs) throw new Error("PDF.js is not available in this build"); - const data = pdfBytes instanceof Uint8Array ? pdfBytes : new Uint8Array(pdfBytes); + const data = + pdfBytes instanceof Uint8Array ? pdfBytes : new Uint8Array(pdfBytes); const pdf = await pdfjs.getDocument({ data }).promise; const pages: RasterPage[] = []; for (let i = 1; i <= pdf.numPages; i++) { diff --git a/frontend/editor/src/core/services/formDetection/preprocess.ts b/frontend/editor/src/core/services/formDetection/preprocess.ts index 41003644af..97c3749be7 100644 --- a/frontend/editor/src/core/services/formDetection/preprocess.ts +++ b/frontend/editor/src/core/services/formDetection/preprocess.ts @@ -2,7 +2,10 @@ // counterpart of Yolo.preprocess. The resize uses a 2D canvas (bilinear-ish); the normalisation // step is split out as a pure function so it can be unit-tested against the Java golden vectors. -import { ModelPipelineSpec, Preprocessed } from "@app/services/formDetection/types"; +import { + ModelPipelineSpec, + Preprocessed, +} from "@app/services/formDetection/types"; function clampByte(v: number): number { return Math.max(0, Math.min(255, Math.round(v))); @@ -89,8 +92,9 @@ export function preprocess( src.height = srcH; const sctx = src.getContext("2d"); if (!sctx) throw new Error("2D canvas context unavailable"); - const buf = - rgba instanceof Uint8ClampedArray ? rgba : new Uint8ClampedArray(rgba); + // Copy into a fresh ArrayBuffer-backed array: ImageData's type rejects the + // Uint8ClampedArray form (TS 5.7 typed-array generics). + const buf = new Uint8ClampedArray(rgba); sctx.putImageData(new ImageData(buf, srcW, srcH), 0, 0); ctx.imageSmoothingEnabled = true; diff --git a/frontend/editor/src/core/services/formDetection/runBrowserPipeline.ts b/frontend/editor/src/core/services/formDetection/runBrowserPipeline.ts index 5996df9975..316a0f5e65 100644 --- a/frontend/editor/src/core/services/formDetection/runBrowserPipeline.ts +++ b/frontend/editor/src/core/services/formDetection/runBrowserPipeline.ts @@ -8,7 +8,10 @@ import { applyFields } from "@app/services/formDetection/applyFields"; import { toPdfPoints } from "@app/services/formDetection/coordinateMapping"; import { decode } from "@app/services/formDetection/decode"; import { loadModelBytes } from "@app/services/formDetection/modelCache"; -import { getSession, runInference } from "@app/services/formDetection/onnxSession"; +import { + getSession, + runInference, +} from "@app/services/formDetection/onnxSession"; import { renderPages } from "@app/services/formDetection/pdfRender"; import { preprocess } from "@app/services/formDetection/preprocess"; import { DetectedField, resolveSpec } from "@app/services/formDetection/types"; diff --git a/frontend/editor/src/core/services/formDetection/types.ts b/frontend/editor/src/core/services/formDetection/types.ts index 1d4048e1f1..c2a9ced9fd 100644 --- a/frontend/editor/src/core/services/formDetection/types.ts +++ b/frontend/editor/src/core/services/formDetection/types.ts @@ -65,7 +65,9 @@ export interface DetectedField { } /** Resolve a catalog entry's pipeline spec, applying the same defaults the backend uses. */ -export function resolveSpec(entry: FormDetectionCatalogEntry): ModelPipelineSpec { +export function resolveSpec( + entry: FormDetectionCatalogEntry, +): ModelPipelineSpec { return { inputSize: entry.inputSize > 0 ? entry.inputSize : 1216, resizeMode: entry.resizeMode ?? "letterbox", diff --git a/frontend/editor/src/core/tests/stubbed/auto-form-detection.spec.ts b/frontend/editor/src/core/tests/stubbed/auto-form-detection.spec.ts index e7241c2f17..ea483e2e7f 100644 --- a/frontend/editor/src/core/tests/stubbed/auto-form-detection.spec.ts +++ b/frontend/editor/src/core/tests/stubbed/auto-form-detection.spec.ts @@ -105,9 +105,7 @@ test.describe("Auto Form Detection tool", () => { await page.goto("/"); const configBtn = page.locator('[data-testid="config-button"]').first(); - if ( - !(await configBtn.isVisible({ timeout: 5_000 }).catch(() => false)) - ) { + if (!(await configBtn.isVisible({ timeout: 5_000 }).catch(() => false))) { test.skip(true, "Config button not rendered for admin on this build"); return; } @@ -119,8 +117,8 @@ test.describe("Auto Form Detection tool", () => { // not as its own nav entry - navigate there first. await dialog.getByText("Features", { exact: true }).first().click(); - await expect( - dialog.getByText(/AI Form Detection/i).first(), - ).toBeVisible({ timeout: 5_000 }); + await expect(dialog.getByText(/AI Form Detection/i).first()).toBeVisible({ + timeout: 5_000, + }); }); }); diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFormDetectionSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFormDetectionSection.tsx index 994c56edff..9387249001 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFormDetectionSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFormDetectionSection.tsx @@ -133,7 +133,9 @@ export default function AdminFormDetectionSection() { doSetConfig({ enabled: e.currentTarget.checked })} + onChange={(e) => + doSetConfig({ enabled: e.currentTarget.checked }) + } disabled={configBusy || (loading && !status)} size="sm" aria-label={t( @@ -155,9 +157,7 @@ export default function AdminFormDetectionSection() { ) : ( - - {t("admin.formDetection.status", "Status")}: - + {t("admin.formDetection.status", "Status")}: {st ?? "unknown"} @@ -182,7 +182,9 @@ export default function AdminFormDetectionSection() { - doSetConfig({ executionMode: v as FormDetectionExecutionMode }) + doSetConfig({ + executionMode: v as FormDetectionExecutionMode, + }) } disabled={configBusy || !enabled} data={[ @@ -267,7 +269,9 @@ export default function AdminFormDetectionSection() { "1. On a machine with internet, download the model file:", )} - {`curl -L -o ${selectedEntry.id}.onnx "${selectedEntry.onnxUrl}"`} + {`curl -L -o ${selectedEntry.id}.onnx "${selectedEntry.onnxUrl}"`} {t( "admin.formDetection.airgap.step2", @@ -276,7 +280,10 @@ export default function AdminFormDetectionSection() { {selectedEntry.sha256 || - t("admin.formDetection.airgap.noSha", "(checksum not set)")} + t( + "admin.formDetection.airgap.noSha", + "(checksum not set)", + )} {t( @@ -284,7 +291,9 @@ export default function AdminFormDetectionSection() { "3. Copy it onto the Stirling-PDF server into the model directory:", )} - {`/models/form-detection/${selectedEntry.id}.onnx`} + {`/models/form-detection/${selectedEntry.id}.onnx`} {t( "admin.formDetection.airgap.step4",