diff --git a/.dockerignore b/.dockerignore index 33be4b6ee1..4988174bbb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -88,6 +88,13 @@ stirling/ customFiles/ configs/ +# Local dev runtime dirs created by bootRun under the module dir (hold the locked H2 DB, +# downloaded models, logs); never build input. Root-level patterns above don't match these. +**/configs/ +app/core/customFiles/ +app/core/logs/ +app/core/pipeline/ + # Claude Code workspace .claude/ diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index c6da76309f..0fbdc6eeeb 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -319,6 +319,14 @@ public class ApplicationProperties { /** Optional override dir; blank uses {@code /models/form-detection}. */ private String modelDir = ""; + + /** + * Read-only dir of models baked into the image (e.g. the Docker server image pre-downloads + * FFDNet-S here). On startup any {@code .onnx} found here is copied into the + * writable model dir if not already present, and activated if no model is active - so the + * feature works out-of-the-box. Blank (default) disables seeding. + */ + private String preinstalledModelDir = ""; } /** diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index bae86827fc..fc44f25f9e 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -369,6 +369,7 @@ formDetection: executionMode: auto # Where detection runs: 'auto' (browser first, server fallback), 'browser' (in-browser WASM only, PDF never leaves the device), or 'server' (backend inference) activeModelId: "" # Id of the installed Auto Form Detection model (set automatically after an admin installs one) modelDir: "" # Optional override directory for downloaded .onnx models; blank uses /models/form-detection + preinstalledModelDir: "" # Read-only dir of models baked into the image (Docker pre-downloads FFDNet-S here); seeded into the model dir on startup. Blank disables. policies: # Folder automations can read from and write to the directories you allow here, so treat this as a diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/formdetection/service/FormDetectionModelManager.java b/app/proprietary/src/main/java/stirling/software/proprietary/formdetection/service/FormDetectionModelManager.java index d7927474ad..dc0e3262e0 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/formdetection/service/FormDetectionModelManager.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/formdetection/service/FormDetectionModelManager.java @@ -92,6 +92,7 @@ public class FormDetectionModelManager { @PostConstruct void init() { sweepTempFiles(); + seedPreinstalledModels(); Optional active = getActiveModelFile(); if (active.isPresent()) { activeSha = getActiveEntry().map(ModelCatalogEntry::getSha256).orElse(null); @@ -417,6 +418,58 @@ public class FormDetectionModelManager { return Paths.get(runtimePathConfig.getFormDetectionModelPath()); } + /** + * Copy any image-baked models (see {@code formDetection.preinstalledModelDir}) into the + * writable model dir if not already present, and activate one when nothing is active yet. Lets + * the Docker server image ship with FFDNet-S ready without an admin install. No-op when the dir + * is unset or missing (desktop/local). + */ + private void seedPreinstalledModels() { + String preDir = applicationProperties.getFormDetection().getPreinstalledModelDir(); + if (StringUtils.isBlank(preDir)) { + return; + } + Path src = Paths.get(preDir); + if (!Files.isDirectory(src)) { + return; + } + Path dir = modelDir(); + try { + Files.createDirectories(dir); + } catch (IOException e) { + log.warn("Cannot create model dir to seed pre-installed models: {}", e.getMessage()); + return; + } + if (!isWritable(dir)) { + log.warn("Model dir {} not writable; skipping pre-installed model seeding", dir); + return; + } + try (DirectoryStream models = Files.newDirectoryStream(src, "*.onnx")) { + for (Path p : models) { + String fn = p.getFileName().toString(); + String id = fn.substring(0, fn.length() - ".onnx".length()); + if (!SAFE_ID.matcher(id).matches() || catalog.getById(id).isEmpty()) { + continue; + } + Path target = dir.resolve(id + ".onnx"); + if (!Files.exists(target)) { + Files.copy(p, target, StandardCopyOption.COPY_ATTRIBUTES); + log.info("Seeded pre-installed Auto Form Detection model '{}'", id); + } + if (StringUtils.isBlank(activeModelId())) { + applicationProperties.getFormDetection().setActiveModelId(id); + try { + GeneralUtils.saveKeyToSettings("formDetection.activeModelId", id); + } catch (IOException e) { + log.warn("Could not persist seeded activeModelId: {}", e.getMessage()); + } + } + } + } catch (IOException e) { + log.warn("Failed to seed pre-installed models from {}: {}", src, e.getMessage()); + } + } + private void sweepTempFiles() { Path dir = modelDir(); if (!Files.isDirectory(dir)) { diff --git a/docker/embedded/Dockerfile b/docker/embedded/Dockerfile index 0049adf373..86377c6397 100644 --- a/docker/embedded/Dockerfile +++ b/docker/embedded/Dockerfile @@ -19,6 +19,15 @@ RUN apt-get update \ && rm /tmp/task.deb \ && rm -rf /var/lib/apt/lists/* +# Pre-download the default Auto Form Detection model (FFDNet-S, ~37MB) so the feature works +# out-of-the-box. Verified by checksum here; seeded into the writable configs model dir at startup +# by FormDetectionModelManager (FORMDETECTION_PREINSTALLEDMODELDIR below). +ARG FORM_DETECTION_MODEL_URL=https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/main/FFDNet-S.onnx +ARG FORM_DETECTION_MODEL_SHA256=93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0 +RUN mkdir -p /preinstalled-models \ + && curl -fSL "${FORM_DETECTION_MODEL_URL}" -o /preinstalled-models/ffdnet-s.onnx \ + && echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/ffdnet-s.onnx" | sha256sum -c - + # JDK 25+: --add-exports is no longer accepted via JAVA_TOOL_OPTIONS; use JDK_JAVA_OPTIONS instead ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ @@ -84,6 +93,9 @@ COPY --link --from=app-build --chown=1000:1000 \ /app/build/libs/restart-helper.jar /restart-helper.jar COPY --link --chown=1000:1000 scripts/ /scripts/ +# Pre-bundled default detection model; FormDetectionModelManager seeds it into /configs on startup. +COPY --link --from=app-build --chown=1000:1000 /preinstalled-models/ /opt/stirling/preinstalled-models/ + # Fonts go to system dir, root ownership is correct (world-readable) COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/truetype/ @@ -107,6 +119,7 @@ RUN echo "${VERSION_TAG:-dev}" > /etc/stirling_version # Environment variables ENV VERSION_TAG=$VERSION_TAG \ + FORMDETECTION_PREINSTALLEDMODELDIR="/opt/stirling/preinstalled-models" \ STIRLING_AOT_ENABLE="false" \ STIRLING_JVM_PROFILE="balanced" \ _JVM_OPTS_BALANCED="-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/configs/heap_dumps -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=4m -XX:G1PeriodicGCInterval=60000 -XX:+UseStringDeduplication -XX:+UseCompactObjectHeaders -XX:+ExplicitGCInvokesConcurrent -Dspring.threads.virtual.enabled=true -Djava.awt.headless=true" \ diff --git a/docker/embedded/Dockerfile.fat b/docker/embedded/Dockerfile.fat index 38fb181b65..a270ab06bc 100644 --- a/docker/embedded/Dockerfile.fat +++ b/docker/embedded/Dockerfile.fat @@ -20,6 +20,15 @@ RUN apt-get update \ && rm /tmp/task.deb \ && rm -rf /var/lib/apt/lists/* +# Pre-download the default Auto Form Detection model (FFDNet-S, ~37MB) so the feature works +# out-of-the-box - especially important for this air-gapped fat image (no runtime internet). +# Verified by checksum; seeded into the writable configs model dir at startup. +ARG FORM_DETECTION_MODEL_URL=https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/main/FFDNet-S.onnx +ARG FORM_DETECTION_MODEL_SHA256=93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0 +RUN mkdir -p /preinstalled-models \ + && curl -fSL "${FORM_DETECTION_MODEL_URL}" -o /preinstalled-models/ffdnet-s.onnx \ + && echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/ffdnet-s.onnx" | sha256sum -c - + # JDK 25+: --add-exports is no longer accepted via JAVA_TOOL_OPTIONS; use JDK_JAVA_OPTIONS instead ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ @@ -79,6 +88,9 @@ COPY --link --from=app-build --chown=1000:1000 \ /app/build/libs/restart-helper.jar /restart-helper.jar COPY --link --chown=1000:1000 scripts/ /scripts/ +# Pre-bundled default detection model; FormDetectionModelManager seeds it into /configs on startup. +COPY --link --from=app-build --chown=1000:1000 /preinstalled-models/ /opt/stirling/preinstalled-models/ + # Fonts go to system dir, root ownership is correct (world-readable) COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/truetype/ @@ -101,6 +113,7 @@ RUN echo "${VERSION_TAG:-dev}" > /etc/stirling_version # Environment variables ENV VERSION_TAG=$VERSION_TAG \ + FORMDETECTION_PREINSTALLEDMODELDIR="/opt/stirling/preinstalled-models" \ STIRLING_AOT_ENABLE="false" \ STIRLING_JVM_PROFILE="balanced" \ _JVM_OPTS_BALANCED="-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/configs/heap_dumps -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=4m -XX:G1PeriodicGCInterval=60000 -XX:+UseStringDeduplication -XX:+UseCompactObjectHeaders -XX:+ExplicitGCInvokesConcurrent -Dspring.threads.virtual.enabled=true -Djava.awt.headless=true" \