Add form-detection i18n keys and fix in-browser engine typing

2026-06-15 10:47:05 +01:00
parent c168e5a658
commit 7bcaeb69c5
11 changed files with 124 additions and 34 deletions
@@ -558,6 +558,39 @@ close = "Close"
 error = "Error"
 expand = "Expand"

+[admin.formDetection]
+active = "Active"
+description = "Install the AI model used to auto-detect form fields. The selected model is downloaded on demand (about 40-100MB) into the configs volume and is not bundled with Stirling-PDF."
+enableFeature = "Enable feature"
+engineDescription = "Browser keeps the PDF on the device (downloads a ~12MB runtime once, then cached); Server runs it on the backend; Auto prefers the browser and falls back to the server."
+engineLabel = "Where detection runs"
+install = "Install"
+license = "License"
+notAvailable = "This catalog entry has no download URL/checksum configured yet, so it cannot be installed."
+notWritable = "The model directory is not writable; check the configs volume mount."
+selectModel = "Model"
+selectPlaceholder = "Select a model"
+status = "Status"
+switch = "Switch to this model"
+title = "AI Form Detection"
+uninstall = "Uninstall"
+
+[admin.formDetection.airgap]
+hide = "Hide offline install instructions"
+intro = "No internet on the server? Install the model manually:"
+noSha = "(checksum not set)"
+show = "Air-gapped / offline install instructions"
+step1 = "1. On a machine with internet, download the model file:"
+step2 = "2. Verify its SHA-256 checksum matches:"
+step3 = "3. Copy it onto the Stirling-PDF server into the model directory:"
+step4 = "4. Set formDetection.activeModelId to this model's id in settings.yml, then restart (an installed model is auto-detected on boot). <configs> is the configs volume (e.g. /configs in Docker). Alternatively, point the Install button at an internal mirror using an override URL + checksum."
+
+[admin.formDetection.engine]
+auto = "Auto"
+browser = "Browser"
+server = "Server"
+serverUnavailable = "The server engine is not bundled in this build, so detection runs in the browser. Use Auto or Browser."
+
 [admin.settings]
 discard = "Discard"
 error = "Failed to save settings"
@@ -1837,6 +1870,16 @@ bullet2 = "Creates a clean, valid filename from the detected title"
 bullet3 = "Keeps the original name if no suitable title is found"
 title = "Smart Renaming"

+[autoFormDetection]
+loading = "Detecting form fields..."
+submit = "Detect & make fillable"
+
+[autoFormDetection.error]
+failed = "An error occurred while detecting form fields."
+
+[autoFormDetection.results]
+title = "Detected Form Fields"
+
 [automate]
 copyToSaved = "Copy to Saved"
 desc = "Build multi-step workflows by chaining together PDF actions. Ideal for recurring tasks."
@@ -59,9 +59,8 @@ async function browserDetect(
 ): Promise<File> {
  // Lazy-load the in-browser engine (onnxruntime-web + the ~12MB wasm) only when browser-mode
  // detection actually runs - it is never pulled into the initial bundle or loaded on the homepage.
-  const { runBrowserDetection } = await import(
-    "@app/services/formDetection/runBrowserPipeline"
-  );
+  const { runBrowserDetection } =
+    await import("@app/services/formDetection/runBrowserPipeline");
  const bytes = await file.arrayBuffer();
  const { appliedPdf } = await runBrowserDetection(
    bytes,
@@ -43,12 +43,24 @@ const pre: Preprocessed = {
 // nc_first layout [channels=6][anchors=3], data[c*anchors + a]
 // box A (cx5,cy5,w4,h4) twice (overlapping) + box B (cx8,cy8,w2,h2)
 const data = [
-  5, 5, 8, // cx
-  5, 5, 8, // cy
-  4, 4, 2, // w
-  4, 4, 2, // h
-  0.9, 0.8, 0.7, // text score
-  0.1, 0.1, 0.1, // choice score
+  5,
+  5,
+  8, // cx
+  5,
+  5,
+  8, // cy
+  4,
+  4,
+  2, // w
+  4,
+  4,
+  2, // h
+  0.9,
+  0.8,
+  0.7, // text score
+  0.1,
+  0.1,
+  0.1, // choice score
 ];
 const out: RawOutput = { data, d1: 6, d2: 3 };

@@ -1,7 +1,12 @@
 // Pure decode/NMS/un-projection - a 1:1 port of Yolo.decode in the backend. Kept free of any
 // browser API so it can be unit-tested for parity against the Java golden output.

-import { Detection, ModelPipelineSpec, Preprocessed, RawOutput } from "@app/services/formDetection/types";
+import {
+  Detection,
+  ModelPipelineSpec,
+  Preprocessed,
+  RawOutput,
+} from "@app/services/formDetection/types";

 function at(
  data: Float32Array | number[],
@@ -30,7 +35,11 @@ function iou(a: Detection, b: Detection): number {
  return union <= 0 ? 0 : inter / union;
 }

-function nms(dets: Detection[], mode: string, iouThreshold: number): Detection[] {
+function nms(
+  dets: Detection[],
+  mode: string,
+  iouThreshold: number,
+): Detection[] {
  if (dets.length < 2 || (mode ?? "").toLowerCase() === "none") {
    return dets;
  }
@@ -78,7 +87,8 @@ export function decode(
    let bestClass = -1;
    let bestScore = 0;
    for (let c = 0; c < numClasses; c++) {
-      const s = at(data, ncFirst, anchors, channels, classOffset + c, a) * objScore;
+      const s =
+        at(data, ncFirst, anchors, channels, classOffset + c, a) * objScore;
      if (s > bestScore) {
        bestScore = s;
        bestClass = c;
@@ -100,7 +110,14 @@ export function decode(
    ow = Math.max(0, Math.min(ow, pre.srcW - cxl));
    oh = Math.max(0, Math.min(oh, pre.srcH - cyl));
    if (ow <= 0 || oh <= 0) continue;
-    dets.push({ classId: bestClass, score: bestScore, x: cxl, y: cyl, w: ow, h: oh });
+    dets.push({
+      classId: bestClass,
+      score: bestScore,
+      x: cxl,
+      y: cyl,
+      w: ow,
+      h: oh,
+    });
  }
  return nms(dets, spec.nms, spec.iou);
 }
@@ -28,7 +28,9 @@ async function verify(bytes: ArrayBuffer, expectedSha?: string): Promise<void> {
 * Return the active model bytes, from the Cache API when present (and checksum-valid) or by
 * downloading from the backend. The cache key is the checksum, so a model swap naturally misses.
 */
-export async function loadModelBytes(expectedSha?: string): Promise<ArrayBuffer> {
+export async function loadModelBytes(
+  expectedSha?: string,
+): Promise<ArrayBuffer> {
  const cacheKey = `${MODEL_FILE_URL}#${expectedSha ?? "nosha"}`;
  // Cache API is unavailable in non-secure contexts; degrade to a plain download in that case.
  const cache = await caches.open(CACHE_NAME).catch(() => null);
@@ -46,7 +46,8 @@ export async function renderPages(
  const pdfjs = window.pdfjsLib;
  if (!pdfjs) throw new Error("PDF.js is not available in this build");

-  const data = pdfBytes instanceof Uint8Array ? pdfBytes : new Uint8Array(pdfBytes);
+  const data =
+    pdfBytes instanceof Uint8Array ? pdfBytes : new Uint8Array(pdfBytes);
  const pdf = await pdfjs.getDocument({ data }).promise;
  const pages: RasterPage[] = [];
  for (let i = 1; i <= pdf.numPages; i++) {
@@ -2,7 +2,10 @@
 // counterpart of Yolo.preprocess. The resize uses a 2D canvas (bilinear-ish); the normalisation
 // step is split out as a pure function so it can be unit-tested against the Java golden vectors.

-import { ModelPipelineSpec, Preprocessed } from "@app/services/formDetection/types";
+import {
+  ModelPipelineSpec,
+  Preprocessed,
+} from "@app/services/formDetection/types";

 function clampByte(v: number): number {
  return Math.max(0, Math.min(255, Math.round(v)));
@@ -89,8 +92,9 @@ export function preprocess(
  src.height = srcH;
  const sctx = src.getContext("2d");
  if (!sctx) throw new Error("2D canvas context unavailable");
-  const buf =
-    rgba instanceof Uint8ClampedArray ? rgba : new Uint8ClampedArray(rgba);
+  // Copy into a fresh ArrayBuffer-backed array: ImageData's type rejects the
+  // Uint8ClampedArray<ArrayBufferLike> form (TS 5.7 typed-array generics).
+  const buf = new Uint8ClampedArray(rgba);
  sctx.putImageData(new ImageData(buf, srcW, srcH), 0, 0);

  ctx.imageSmoothingEnabled = true;
@@ -8,7 +8,10 @@ import { applyFields } from "@app/services/formDetection/applyFields";
 import { toPdfPoints } from "@app/services/formDetection/coordinateMapping";
 import { decode } from "@app/services/formDetection/decode";
 import { loadModelBytes } from "@app/services/formDetection/modelCache";
-import { getSession, runInference } from "@app/services/formDetection/onnxSession";
+import {
+  getSession,
+  runInference,
+} from "@app/services/formDetection/onnxSession";
 import { renderPages } from "@app/services/formDetection/pdfRender";
 import { preprocess } from "@app/services/formDetection/preprocess";
 import { DetectedField, resolveSpec } from "@app/services/formDetection/types";
@@ -65,7 +65,9 @@ export interface DetectedField {
 }

 /** Resolve a catalog entry's pipeline spec, applying the same defaults the backend uses. */
-export function resolveSpec(entry: FormDetectionCatalogEntry): ModelPipelineSpec {
+export function resolveSpec(
+  entry: FormDetectionCatalogEntry,
+): ModelPipelineSpec {
  return {
    inputSize: entry.inputSize > 0 ? entry.inputSize : 1216,
    resizeMode: entry.resizeMode ?? "letterbox",
@@ -105,9 +105,7 @@ test.describe("Auto Form Detection tool", () => {
    await page.goto("/");

    const configBtn = page.locator('[data-testid="config-button"]').first();
-    if (
-      !(await configBtn.isVisible({ timeout: 5_000 }).catch(() => false))
-    ) {
+    if (!(await configBtn.isVisible({ timeout: 5_000 }).catch(() => false))) {
      test.skip(true, "Config button not rendered for admin on this build");
      return;
    }
@@ -119,8 +117,8 @@ test.describe("Auto Form Detection tool", () => {
    // not as its own nav entry - navigate there first.
    await dialog.getByText("Features", { exact: true }).first().click();

-    await expect(
-      dialog.getByText(/AI Form Detection/i).first(),
-    ).toBeVisible({ timeout: 5_000 });
+    await expect(dialog.getByText(/AI Form Detection/i).first()).toBeVisible({
+      timeout: 5_000,
+    });
  });
 });
@@ -133,7 +133,9 @@ export default function AdminFormDetectionSection() {
            </Text>
            <Switch
              checked={enabled}
-              onChange={(e) => doSetConfig({ enabled: e.currentTarget.checked })}
+              onChange={(e) =>
+                doSetConfig({ enabled: e.currentTarget.checked })
+              }
              disabled={configBusy || (loading && !status)}
              size="sm"
              aria-label={t(
@@ -155,9 +157,7 @@ export default function AdminFormDetectionSection() {
        ) : (
          <Stack gap="sm">
            <Group gap="xs">
-              <Text fw={500}>
-                {t("admin.formDetection.status", "Status")}:
-              </Text>
+              <Text fw={500}>{t("admin.formDetection.status", "Status")}:</Text>
              <Badge color={badgeColor(st)} variant="light" size="sm">
                {st ?? "unknown"}
              </Badge>
@@ -182,7 +182,9 @@ export default function AdminFormDetectionSection() {
              <SegmentedControl
                value={executionMode}
                onChange={(v) =>
-                  doSetConfig({ executionMode: v as FormDetectionExecutionMode })
+                  doSetConfig({
+                    executionMode: v as FormDetectionExecutionMode,
+                  })
                }
                disabled={configBusy || !enabled}
                data={[
@@ -267,7 +269,9 @@ export default function AdminFormDetectionSection() {
                      "1. On a machine with internet, download the model file:",
                    )}
                  </Text>
-                  <Code block>{`curl -L -o ${selectedEntry.id}.onnx "${selectedEntry.onnxUrl}"`}</Code>
+                  <Code
+                    block
+                  >{`curl -L -o ${selectedEntry.id}.onnx "${selectedEntry.onnxUrl}"`}</Code>
                  <Text size="xs">
                    {t(
                      "admin.formDetection.airgap.step2",
@@ -276,7 +280,10 @@ export default function AdminFormDetectionSection() {
                  </Text>
                  <Code block>
                    {selectedEntry.sha256 ||
-                      t("admin.formDetection.airgap.noSha", "(checksum not set)")}
+                      t(
+                        "admin.formDetection.airgap.noSha",
+                        "(checksum not set)",
+                      )}
                  </Code>
                  <Text size="xs">
                    {t(
@@ -284,7 +291,9 @@ export default function AdminFormDetectionSection() {
                      "3. Copy it onto the Stirling-PDF server into the model directory:",
                    )}
                  </Text>
-                  <Code block>{`<configs>/models/form-detection/${selectedEntry.id}.onnx`}</Code>
+                  <Code
+                    block
+                  >{`<configs>/models/form-detection/${selectedEntry.id}.onnx`}</Code>
                  <Text size="xs" c="dimmed">
                    {t(
                      "admin.formDetection.airgap.step4",