Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7164bff8e3 | ||
|
|
793cefa5ac | ||
|
|
b95b7c53c7 | ||
|
|
a9641bf583 | ||
|
|
9e7661eedd | ||
|
|
ba9eeddf59 | ||
|
|
a49b2275e7 | ||
|
|
a660aed964 | ||
|
|
1077c4f134 | ||
|
|
e468ed5e65 | ||
|
|
bad9e9ac61 | ||
|
|
4da5b4b7eb | ||
|
|
26d835eda6 | ||
|
|
cfd6d19886 | ||
|
|
7bcaeb69c5 | ||
|
|
c168e5a658 | ||
|
|
313f614ec2 | ||
|
|
b0109e174b | ||
|
|
38180550a3 |
@@ -87,6 +87,13 @@ stirling/
|
||||
customFiles/
|
||||
configs/
|
||||
|
||||
# Local dev runtime dirs created by bootRun under the module dir (hold the locked H2 DB,
|
||||
# downloaded models, logs); never build input. Root-level patterns above don't match these.
|
||||
**/configs/
|
||||
app/core/customFiles/
|
||||
app/core/logs/
|
||||
app/core/pipeline/
|
||||
|
||||
# Claude Code workspace
|
||||
.claude/
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.14.2
|
||||
pkgver=2.14.1
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.14.2
|
||||
pkgver=2.14.1
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -510,7 +510,6 @@ jobs:
|
||||
# cargo output unsigned, so checking it produces false negatives.
|
||||
- name: Verify Windows Code Signature
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master') }}
|
||||
timeout-minutes: 15
|
||||
shell: pwsh
|
||||
run: |
|
||||
$allSigned = $true
|
||||
@@ -532,26 +531,11 @@ jobs:
|
||||
|
||||
# Extract MSI and verify the inner exe (the file that actually gets installed).
|
||||
# This is the critical check - AV flags the installed exe at runtime.
|
||||
# Use lessmsi, not `msiexec /a`: msiexec serializes on the global
|
||||
# _MSIExecute mutex and hangs forever on hosted runners when another
|
||||
# installer is busy. lessmsi reads MSI tables directly - no mutex, no service.
|
||||
$msi = $msiFiles[0].FullName
|
||||
$extractDir = Join-Path $env:RUNNER_TEMP "msi-verify"
|
||||
if (Test-Path $extractDir) { Remove-Item $extractDir -Recurse -Force }
|
||||
New-Item -ItemType Directory -Force -Path $extractDir | Out-Null
|
||||
|
||||
choco install lessmsi -y --no-progress --limit-output | Out-Null
|
||||
|
||||
# Bound the extraction and kill on hang (defence in depth over timeout-minutes).
|
||||
$proc = Start-Process lessmsi -ArgumentList 'x', "`"$msi`"", "`"$extractDir\`"" -PassThru -NoNewWindow
|
||||
if (-not $proc.WaitForExit(120000)) {
|
||||
try { $proc.Kill() } catch {}
|
||||
Write-Host "[ERROR] MSI extraction timed out after 120s"
|
||||
$allSigned = $false
|
||||
} elseif ($proc.ExitCode -ne 0) {
|
||||
Write-Host "[ERROR] Failed to extract MSI for verification (exit code: $($proc.ExitCode))"
|
||||
$allSigned = $false
|
||||
} else {
|
||||
$proc = Start-Process msiexec.exe -ArgumentList '/a', $msi, '/qn', "TARGETDIR=$extractDir" -Wait -PassThru -NoNewWindow
|
||||
if ($proc.ExitCode -eq 0) {
|
||||
$innerExe = Get-ChildItem -Path $extractDir -Filter "stirling-pdf.exe" -Recurse -File | Select-Object -First 1
|
||||
if ($innerExe) {
|
||||
$sig = Get-AuthenticodeSignature -FilePath $innerExe.FullName
|
||||
@@ -564,6 +548,9 @@ jobs:
|
||||
Write-Host "[ERROR] Could not find stirling-pdf.exe inside MSI"
|
||||
$allSigned = $false
|
||||
}
|
||||
} else {
|
||||
Write-Host "[ERROR] Failed to extract MSI for verification (exit code: $($proc.ExitCode))"
|
||||
$allSigned = $false
|
||||
}
|
||||
|
||||
if (-not $allSigned) {
|
||||
@@ -813,11 +800,7 @@ jobs:
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
|
||||
with:
|
||||
tag_name: v${{ needs.determine-matrix.outputs.version }}
|
||||
# Don't regenerate/append notes on re-runs, and don't force this into the
|
||||
# "Latest" slot - leave the release body and latest marker as they are.
|
||||
generate_release_notes: false
|
||||
append_body: false
|
||||
make_latest: false
|
||||
generate_release_notes: true
|
||||
fail_on_unmatched_files: true
|
||||
# Installers + updater payloads + manifest. .sig contents are embedded
|
||||
# in latest.json so the .sig files themselves are not uploaded.
|
||||
|
||||
@@ -208,18 +208,6 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "The W3C License"
|
||||
},
|
||||
{
|
||||
"moduleName": "com.google.re2j:re2j",
|
||||
"moduleLicense": "Go License"
|
||||
},
|
||||
{
|
||||
"moduleName": "com.hubspot:algebra",
|
||||
"moduleLicense": null
|
||||
},
|
||||
{
|
||||
"moduleName": "com.hubspot.immutables:immutables-exceptions",
|
||||
"moduleLicense": null
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "UnRar License"
|
||||
|
||||
@@ -536,6 +536,7 @@ public class EndpointConfiguration {
|
||||
addEndpointToGroup("Java", "pdf-to-epub");
|
||||
addEndpointToGroup("Java", "eml-to-pdf");
|
||||
addEndpointToGroup("Java", "handleData");
|
||||
addEndpointToGroup("Java", "form-detection");
|
||||
addEndpointToGroup("rar", "pdf-to-cbr");
|
||||
|
||||
// Javascript
|
||||
|
||||
+19
@@ -40,6 +40,9 @@ public class RuntimePathConfig {
|
||||
// Tesseract data path
|
||||
private final String tessDataPath;
|
||||
|
||||
// Auto Form Detection model directory
|
||||
private final String formDetectionModelPath;
|
||||
|
||||
private final List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> unoServerEndpoints;
|
||||
|
||||
// Pipeline paths
|
||||
@@ -130,6 +133,22 @@ public class RuntimePathConfig {
|
||||
|
||||
log.info("Using Tesseract data path: {}", this.tessDataPath);
|
||||
|
||||
// Auto Form Detection model directory (kept under <configs> so it survives
|
||||
// restarts/updates)
|
||||
String configuredModelDir =
|
||||
properties.getFormDetection() != null
|
||||
? properties.getFormDetection().getModelDir()
|
||||
: null;
|
||||
this.formDetectionModelPath =
|
||||
StringUtils.isNotBlank(configuredModelDir)
|
||||
? configuredModelDir
|
||||
: Path.of(
|
||||
InstallationPathConfig.getConfigPath(),
|
||||
"models",
|
||||
"form-detection")
|
||||
.toString();
|
||||
log.info("Using Auto Form Detection model path: {}", this.formDetectionModelPath);
|
||||
|
||||
ApplicationProperties.ProcessExecutor processExecutor = properties.getProcessExecutor();
|
||||
int libreOfficeLimit = 1;
|
||||
if (processExecutor != null && processExecutor.getSessionLimit() != null) {
|
||||
|
||||
@@ -77,6 +77,7 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private FormDetection formDetection = new FormDetection();
|
||||
private Mcp mcp = new Mcp();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
@@ -310,6 +311,37 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto Form Detection settings. The model itself is downloaded on demand by an admin (see
|
||||
* {@code /api/v1/ai/form-detection-model/*}); only lightweight pointers are persisted here.
|
||||
*/
|
||||
@Data
|
||||
public static class FormDetection {
|
||||
/** Master on/off switch for the whole feature (admin-controlled). */
|
||||
private boolean enabled = true;
|
||||
|
||||
/**
|
||||
* Where detection runs: {@code auto} (browser first, server fallback), {@code browser}
|
||||
* (in-browser WASM only - the PDF never leaves the device), or {@code server} (backend
|
||||
* inference). Read by the frontend tool to choose its pipeline.
|
||||
*/
|
||||
private String executionMode = "auto";
|
||||
|
||||
/** Id of the installed model; blank means none installed. */
|
||||
private String activeModelId = "";
|
||||
|
||||
/** Optional override dir; blank uses {@code <configs>/models/form-detection}. */
|
||||
private String modelDir = "";
|
||||
|
||||
/**
|
||||
* Read-only dir of models baked into the image (e.g. the Docker server image pre-downloads
|
||||
* FFDNet-S here). On startup any {@code <catalogId>.onnx} found here is copied into the
|
||||
* writable model dir if not already present, and activated if no model is active - so the
|
||||
* feature works out-of-the-box. Blank (default) disables seeding.
|
||||
*/
|
||||
private String preinstalledModelDir = "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Model Context Protocol (MCP) server configuration. All keys live under the top-level {@code
|
||||
* mcp.*} prefix. {@link #enabled} defaults to {@code false}: when off, no MCP beans are wired,
|
||||
|
||||
@@ -944,6 +944,81 @@ public class FormUtils {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create new AcroForm fields from a list of definitions (used by Auto Form Detection). Reuses
|
||||
* the same field-creation and appearance logic as the rest of this class, and creates the
|
||||
* AcroForm (with a Helvetica default resource) when the document has none. Field names are made
|
||||
* unique against any existing fields.
|
||||
*/
|
||||
public void addFields(PDDocument document, List<NewFormFieldDefinition> definitions)
|
||||
throws IOException {
|
||||
if (document == null || definitions == null || definitions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
PDDocumentCatalog documentCatalog = document.getDocumentCatalog();
|
||||
PDAcroForm acroForm = documentCatalog.getAcroForm();
|
||||
if (acroForm == null) {
|
||||
acroForm = new PDAcroForm(document);
|
||||
PDResources dr = new PDResources();
|
||||
dr.put(COSName.getPDFName("Helv"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
acroForm.setDefaultResources(dr);
|
||||
acroForm.setNeedAppearances(true);
|
||||
documentCatalog.setAcroForm(acroForm);
|
||||
}
|
||||
|
||||
Set<String> existingNames = new java.util.HashSet<>();
|
||||
for (PDField field : acroForm.getFieldTree()) {
|
||||
if (field.getPartialName() != null) {
|
||||
existingNames.add(field.getPartialName());
|
||||
}
|
||||
}
|
||||
|
||||
int pageCount = document.getNumberOfPages();
|
||||
for (NewFormFieldDefinition definition : definitions) {
|
||||
Integer pageIndex = definition.pageIndex();
|
||||
if (pageIndex == null
|
||||
|| pageIndex < 0
|
||||
|| pageIndex >= pageCount
|
||||
|| definition.x() == null
|
||||
|| definition.y() == null
|
||||
|| definition.width() == null
|
||||
|| definition.height() == null) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDRectangle rectangle =
|
||||
new PDRectangle(
|
||||
definition.x(),
|
||||
definition.y(),
|
||||
definition.width(),
|
||||
definition.height());
|
||||
FormFieldTypeSupport handler = FormFieldTypeSupport.forTypeName(definition.type());
|
||||
if (handler == null || handler.doesNotsupportsDefinitionCreation()) {
|
||||
handler = FormFieldTypeSupport.TEXT;
|
||||
}
|
||||
String baseName =
|
||||
(definition.name() != null && !definition.name().isBlank())
|
||||
? definition.name()
|
||||
: handler.typeName() + "_" + (pageIndex + 1);
|
||||
String uniqueName = generateUniqueFieldName(baseName, existingNames);
|
||||
existingNames.add(uniqueName);
|
||||
try {
|
||||
createNewField(
|
||||
handler,
|
||||
acroForm,
|
||||
page,
|
||||
rectangle,
|
||||
uniqueName,
|
||||
definition,
|
||||
definition.options());
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to create detected field '{}': {}", uniqueName, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
ensureAppearances(acroForm);
|
||||
}
|
||||
|
||||
public String filterSingleChoiceSelection(
|
||||
String selection, List<String> allowedOptions, String fieldName) {
|
||||
if (selection == null || selection.trim().isEmpty()) return null;
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDCheckBox;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDTextField;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.util.FormUtils.NewFormFieldDefinition;
|
||||
|
||||
class FormUtilsAddFieldsTest {
|
||||
|
||||
private NewFormFieldDefinition def(String type, int page, float x, float y, float w, float h) {
|
||||
return new NewFormFieldDefinition(
|
||||
null, null, type, page, x, y, w, h, false, null, null, null, null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsTextAndCheckboxFieldsOnPagelessDocument() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
doc.addPage(new PDPage(new PDRectangle(612, 792)));
|
||||
|
||||
FormUtils.addFields(
|
||||
doc,
|
||||
List.of(
|
||||
def("text", 0, 100f, 700f, 200f, 20f),
|
||||
def("checkbox", 0, 100f, 650f, 15f, 15f)));
|
||||
|
||||
PDAcroForm form = doc.getDocumentCatalog().getAcroForm();
|
||||
assertNotNull(form, "AcroForm should be created");
|
||||
|
||||
List<PDField> fields = new ArrayList<>();
|
||||
form.getFieldTree().forEach(fields::add);
|
||||
assertEquals(2, fields.size());
|
||||
|
||||
boolean hasText = fields.stream().anyMatch(f -> f instanceof PDTextField);
|
||||
boolean hasCheckbox = fields.stream().anyMatch(f -> f instanceof PDCheckBox);
|
||||
assertTrue(hasText, "expected a text field");
|
||||
assertTrue(hasCheckbox, "expected a checkbox field");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsOutOfRangePageAndKeepsNamesUnique() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
doc.addPage(new PDPage(new PDRectangle(612, 792)));
|
||||
|
||||
FormUtils.addFields(
|
||||
doc,
|
||||
List.of(
|
||||
def("text", 0, 10f, 10f, 50f, 12f),
|
||||
def("text", 0, 10f, 40f, 50f, 12f),
|
||||
def("text", 5, 10f, 70f, 50f, 12f))); // page 5 out of range -> skipped
|
||||
|
||||
PDAcroForm form = doc.getDocumentCatalog().getAcroForm();
|
||||
List<PDField> fields = new ArrayList<>();
|
||||
form.getFieldTree().forEach(fields::add);
|
||||
assertEquals(2, fields.size());
|
||||
|
||||
long distinctNames = fields.stream().map(PDField::getPartialName).distinct().count();
|
||||
assertEquals(2, distinctNames, "field names must be unique");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void noOpOnEmptyDefinitions() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
doc.addPage(new PDPage(new PDRectangle(612, 792)));
|
||||
FormUtils.addFields(doc, List.of());
|
||||
// no AcroForm forced into existence when there is nothing to add
|
||||
assertEquals(null, doc.getDocumentCatalog().getAcroForm());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -367,6 +367,13 @@ aiEngine:
|
||||
url: http://localhost:5001 # URL of the Python AI engine
|
||||
timeoutSeconds: 120 # Timeout in seconds for AI engine requests
|
||||
|
||||
formDetection:
|
||||
enabled: true # Master on/off switch for the Auto Form Detection feature
|
||||
executionMode: auto # Where detection runs: 'auto' (browser first, server fallback), 'browser' (in-browser WASM only, PDF never leaves the device), or 'server' (backend inference)
|
||||
activeModelId: "" # Id of the installed Auto Form Detection model (set automatically after an admin installs one)
|
||||
modelDir: "" # Optional override directory for downloaded .onnx models; blank uses <configs>/models/form-detection
|
||||
preinstalledModelDir: "" # Read-only dir of models baked into the image (Docker pre-downloads FFDNet-S here); seeded into the model dir on startup. Blank disables.
|
||||
|
||||
policies:
|
||||
# Folder automations can read from and write to the directories you allow here, so treat this as a
|
||||
# security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs
|
||||
|
||||
@@ -3,6 +3,14 @@ repositories {
|
||||
maven { url = "https://build.shibboleth.net/maven/releases" }
|
||||
}
|
||||
|
||||
ext {
|
||||
// CPU build; self-extracting natives for linux/win/mac x64+arm64. Keep major.minor aligned
|
||||
// with the frontend onnxruntime-web pin so the same .onnx runs identically on both paths.
|
||||
// Must be >=1.21 to load opset-22 models: the FFDNet exports are stamped ai.onnx opset 22,
|
||||
// which 1.19/1.20 hard-reject (ORT_FAIL "support is till opset 21"). 1.26.0 matches the
|
||||
// onnxruntime build the upstream CommonForms inference reference runs on.
|
||||
onnxruntimeVersion = '1.26.0'
|
||||
}
|
||||
bootRun {
|
||||
enabled = false
|
||||
}
|
||||
@@ -66,18 +74,16 @@ dependencies {
|
||||
|
||||
implementation "com.google.code.gson:gson:${gsonVersion}"
|
||||
|
||||
// jinjava/jjwt transitively request older Jackson 2 versions; declare the current
|
||||
// version directly so it is selected consistently (root build.gradle pins are the fallback).
|
||||
runtimeOnly "com.fasterxml.jackson.core:jackson-core:${jackson2Version}"
|
||||
runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${jackson2Version}"
|
||||
|
||||
implementation("com.hubspot.jinjava:jinjava:${jinjavaVersion}") {
|
||||
// Compile-time-only annotation artifacts (class-retention annotations, not needed at
|
||||
// runtime) whose declared licences (LGPL / none) fail the licence compatibility check.
|
||||
exclude group: 'com.google.code.findbugs', module: 'annotations'
|
||||
exclude group: 'org.derive4j', module: 'derive4j-annotation'
|
||||
exclude group: 'com.hubspot.immutables', module: 'hubspot-style'
|
||||
exclude group: 'com.hubspot.immutables', module: 'immutable-collection-encodings'
|
||||
// ONNX Runtime (Java) for SERVER-SIDE Auto Form Detection inference.
|
||||
// Compiled against everywhere so the module always builds, but the multi-platform native
|
||||
// (~42MB) is only BUNDLED when explicitly requested via -PbundleOnnxRuntime=true - i.e. the
|
||||
// Docker server image (which then slims it to one Linux arch, ~8MB). Desktop/local/core builds
|
||||
// ship WITHOUT it: server-side detection cleanly reports "unavailable" and the in-browser
|
||||
// (onnxruntime-web) engine handles detection instead. Tests always get it.
|
||||
compileOnly "com.microsoft.onnxruntime:onnxruntime:$onnxruntimeVersion"
|
||||
testImplementation "com.microsoft.onnxruntime:onnxruntime:$onnxruntimeVersion"
|
||||
if ((project.findProperty('bundleOnnxRuntime') ?: 'false').toString().toBoolean()) {
|
||||
runtimeOnly "com.microsoft.onnxruntime:onnxruntime:$onnxruntimeVersion"
|
||||
}
|
||||
|
||||
api 'io.micrometer:micrometer-registry-prometheus'
|
||||
|
||||
-61
@@ -1,61 +0,0 @@
|
||||
package stirling.software.proprietary.classification;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Supplies the classification vocabulary the classify tool sends to the AI engine. The set is a
|
||||
* fixed, built-in list bundled with the application ({@code
|
||||
* classification/classification-labels.json}) and shared by everyone — there is no per-team
|
||||
* customization or database. Loaded once at startup.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class ClassificationLabelProvider {
|
||||
|
||||
private static final String RESOURCE = "classification/classification-labels.json";
|
||||
|
||||
private final List<ClassificationLabel> labels;
|
||||
|
||||
// Explicit @Autowired: the class has a second (private) constructor for tests, so Spring
|
||||
// can't infer which to use without it.
|
||||
@Autowired
|
||||
public ClassificationLabelProvider(ObjectMapper objectMapper) {
|
||||
this(load(objectMapper));
|
||||
}
|
||||
|
||||
private ClassificationLabelProvider(List<ClassificationLabel> labels) {
|
||||
this.labels = List.copyOf(labels);
|
||||
}
|
||||
|
||||
/** Build a provider with an explicit label set (tests). */
|
||||
public static ClassificationLabelProvider withLabels(List<ClassificationLabel> labels) {
|
||||
return new ClassificationLabelProvider(labels);
|
||||
}
|
||||
|
||||
/** The built-in vocabulary, in file order. */
|
||||
public List<ClassificationLabel> labels() {
|
||||
return labels;
|
||||
}
|
||||
|
||||
private static List<ClassificationLabel> load(ObjectMapper objectMapper) {
|
||||
try (InputStream in = new ClassPathResource(RESOURCE).getInputStream()) {
|
||||
ClassificationLabels parsed = objectMapper.readValue(in, ClassificationLabels.class);
|
||||
return parsed.labels();
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to load classification labels from {}", RESOURCE, e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package stirling.software.proprietary.classification;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
import stirling.software.proprietary.classification.model.LabelsValidator;
|
||||
import stirling.software.proprietary.classification.store.ClassificationLabelStore;
|
||||
import stirling.software.proprietary.classification.store.TeamLabelsEntity;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
|
||||
/**
|
||||
* Read/write the team's classification label set — the flat vocabulary the document classifier runs
|
||||
* against. Shared and team-scoped exactly like policies: every user reads their own team's labels,
|
||||
* and only a user who may edit policies (a team leader on SaaS, the global admin self-hosted; see
|
||||
* {@link PolicyManagementAuthority}) may change it — gated only when login is enabled, since
|
||||
* single-user deployments trust the local operator. A team with no stored labels reads as {@code
|
||||
* 204}; that team has no vocabulary, so its documents are not classified (there is no built-in
|
||||
* default on the backend or the engine — the label data lives only in the frontend).
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/classification/labels")
|
||||
@Hidden
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Classification", description = "Team-scoped document-classification labels")
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class ClassificationLabelsController {
|
||||
|
||||
private final ClassificationLabelStore labelStore;
|
||||
private final PolicyManagementAuthority policyManagementAuthority;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final UserServiceInterface userService;
|
||||
|
||||
@GetMapping
|
||||
@Operation(
|
||||
summary = "Get the team's classification labels",
|
||||
description =
|
||||
"Returns the caller's team label set, or 204 when the team has none (its"
|
||||
+ " documents are then not classified).")
|
||||
public ResponseEntity<ClassificationLabels> getTeamLabels() {
|
||||
return labelStore
|
||||
.findByTeam(currentTeamId())
|
||||
.map(ResponseEntity::ok)
|
||||
.orElseGet(() -> ResponseEntity.noContent().build());
|
||||
}
|
||||
|
||||
@PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
summary = "Save the team's classification labels",
|
||||
description =
|
||||
"Validates and stores the label set for the caller's team, shared by everyone"
|
||||
+ " on the team. Requires the policy-editor role for the team.")
|
||||
public ResponseEntity<ClassificationLabels> saveTeamLabels(
|
||||
@RequestBody ClassificationLabels labels) {
|
||||
requireEditingAllowed();
|
||||
validate(labels);
|
||||
ClassificationLabels saved = labelStore.save(currentTeamId(), labels, currentUsername());
|
||||
return ResponseEntity.ok(saved);
|
||||
}
|
||||
|
||||
@DeleteMapping
|
||||
@Operation(
|
||||
summary = "Reset the team's classification labels",
|
||||
description =
|
||||
"Removes the team's stored label set; its documents are then not classified"
|
||||
+ " until labels are saved again. Requires the policy-editor role for the"
|
||||
+ " team.")
|
||||
public ResponseEntity<Void> resetTeamLabels() {
|
||||
requireEditingAllowed();
|
||||
labelStore.deleteByTeam(currentTeamId());
|
||||
return ResponseEntity.noContent().build();
|
||||
}
|
||||
|
||||
private static void validate(ClassificationLabels labels) {
|
||||
try {
|
||||
LabelsValidator.validate(labels);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Editing the team labels requires the editor role for the caller's team — the same gate
|
||||
* policies use (team leader on SaaS, global admin self-hosted). Single-user deployments (login
|
||||
* disabled) have no such role, so they trust the local operator.
|
||||
*/
|
||||
private void requireEditingAllowed() {
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()) {
|
||||
return;
|
||||
}
|
||||
if (!policyManagementAuthority.canEditPolicies()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.FORBIDDEN,
|
||||
"The team classification labels may only be changed by a team leader");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The caller's team key. With login disabled the single operator owns the {@link
|
||||
* TeamLabelsEntity#NO_TEAM} sentinel row; with login enabled a caller with no resolvable team
|
||||
* is an error rather than being dropped into the shared sentinel bucket (which would let
|
||||
* unteamed users read and overwrite each other's "team" labels).
|
||||
*/
|
||||
private Long currentTeamId() {
|
||||
Long teamId = policyManagementAuthority.currentUserTeamId();
|
||||
if (teamId != null) {
|
||||
return teamId;
|
||||
}
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()) {
|
||||
return TeamLabelsEntity.NO_TEAM;
|
||||
}
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.UNAUTHORIZED, "Could not resolve the current user's team");
|
||||
}
|
||||
|
||||
private String currentUsername() {
|
||||
return userService == null ? null : userService.getCurrentUsername();
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -3,10 +3,10 @@ package stirling.software.proprietary.classification.model;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A flat multi-label classification vocabulary — the set of labels a document may be assigned. The
|
||||
* classifier runs against these label names. The vocabulary is a fixed, built-in set shared by
|
||||
* everyone (see {@link stirling.software.proprietary.classification.ClassificationLabelProvider});
|
||||
* this record is the JSON parse target for that bundled resource.
|
||||
* A flat multi-label classification vocabulary — the set of labels a document may be assigned.
|
||||
* Stored per team (admin-edited, shared by everyone on the team); the classifier runs against these
|
||||
* label names. A team with no stored set has no vocabulary, so its documents are not classified —
|
||||
* neither the backend nor the engine holds a default of its own.
|
||||
*/
|
||||
public record ClassificationLabels(List<ClassificationLabel> labels) {
|
||||
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package stirling.software.proprietary.classification.model;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Structural validation for a user- or admin-supplied label set, run before it is stored so a
|
||||
* malformed vocabulary can never reach the classifier. Mirrors the invariants the engine relies on:
|
||||
* non-blank ids and names, each unique within the set (ids exactly, names case-insensitively).
|
||||
*/
|
||||
public final class LabelsValidator {
|
||||
|
||||
private LabelsValidator() {}
|
||||
|
||||
// Generous upper bounds so a legitimate label set is never blocked, but a single team or user
|
||||
// can't store an unbounded blob that would bloat the row, balloon the classifier prompt, or
|
||||
// exhaust memory on deserialize.
|
||||
static final int MAX_LABELS = 500;
|
||||
static final int MAX_TEXT_LENGTH = 128;
|
||||
|
||||
// Icon is a Material Symbols key (lowercase, digits, hyphens). Enforce the SHAPE server-side —
|
||||
// the exact allowlist lives in the frontend — so a client bypassing the UI can't store
|
||||
// arbitrary
|
||||
// text that would render as garbage (or worse) in every teammate's sidebar.
|
||||
private static final Pattern ICON_KEY = Pattern.compile("^[a-z0-9-]+$");
|
||||
|
||||
/**
|
||||
* @throws IllegalArgumentException with a human-readable message when the label set is invalid.
|
||||
*/
|
||||
public static void validate(ClassificationLabels labels) {
|
||||
if (labels == null || labels.labels() == null) {
|
||||
throw new IllegalArgumentException("Labels are required");
|
||||
}
|
||||
if (labels.labels().size() > MAX_LABELS) {
|
||||
throw new IllegalArgumentException("Too many labels (max " + MAX_LABELS + ")");
|
||||
}
|
||||
Set<String> ids = new HashSet<>();
|
||||
Set<String> names = new HashSet<>();
|
||||
for (ClassificationLabel label : labels.labels()) {
|
||||
requireText(label.id(), "Label id");
|
||||
requireText(label.name(), "Label name");
|
||||
if (label.icon() != null && !label.icon().isEmpty()) {
|
||||
if (label.icon().length() > MAX_TEXT_LENGTH) {
|
||||
throw new IllegalArgumentException(
|
||||
"Label icon is too long (max " + MAX_TEXT_LENGTH + " characters)");
|
||||
}
|
||||
if (!ICON_KEY.matcher(label.icon()).matches()) {
|
||||
throw new IllegalArgumentException("Invalid label icon: " + label.icon());
|
||||
}
|
||||
}
|
||||
if (!ids.add(label.id().trim())) {
|
||||
throw new IllegalArgumentException("Duplicate label id: " + label.id());
|
||||
}
|
||||
if (!names.add(label.name().trim().toLowerCase(Locale.ROOT))) {
|
||||
throw new IllegalArgumentException("Duplicate label name: " + label.name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void requireText(String value, String field) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(field + " must not be blank");
|
||||
}
|
||||
if (value.trim().length() > MAX_TEXT_LENGTH) {
|
||||
throw new IllegalArgumentException(
|
||||
field + " is too long (max " + MAX_TEXT_LENGTH + " characters)");
|
||||
}
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
/**
|
||||
* Stores one {@link ClassificationLabels} set per team. A {@code null} teamId addresses the
|
||||
* unteamed set (login disabled / no resolvable team), mirroring how the policy store treats a null
|
||||
* team.
|
||||
*/
|
||||
public interface ClassificationLabelStore {
|
||||
|
||||
/** The team's stored labels, or empty when it has none (callers then skip classification). */
|
||||
Optional<ClassificationLabels> findByTeam(Long teamId);
|
||||
|
||||
/** Create or replace the team's labels. Returns the stored value. */
|
||||
ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy);
|
||||
|
||||
/** Remove the team's labels (reset to default). Returns whether a set existed. */
|
||||
boolean deleteByTeam(Long teamId);
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
/**
|
||||
* In-memory {@link ClassificationLabelStore} for tests and any future no-database mode. {@link
|
||||
* JpaClassificationLabelStore} is the runtime bean.
|
||||
*/
|
||||
public class InProcessClassificationLabelStore implements ClassificationLabelStore {
|
||||
|
||||
private final Map<Long, ClassificationLabels> byTeam = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public Optional<ClassificationLabels> findByTeam(Long teamId) {
|
||||
return Optional.ofNullable(byTeam.get(key(teamId)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) {
|
||||
byTeam.put(key(teamId), labels);
|
||||
return labels;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteByTeam(Long teamId) {
|
||||
return byTeam.remove(key(teamId)) != null;
|
||||
}
|
||||
|
||||
private static long key(Long teamId) {
|
||||
return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId;
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Durable {@link ClassificationLabelStore} backed by JPA; the runtime store. Gated on {@code
|
||||
* policies.enabled} — stored labels only matter when the Classification policy can run — so it
|
||||
* shares the policy subsystem's on/off switch. Each label set is persisted as JSON via {@link
|
||||
* TeamLabelsEntity}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class JpaClassificationLabelStore implements ClassificationLabelStore {
|
||||
|
||||
private final TeamLabelsRepository teamRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public Optional<ClassificationLabels> findByTeam(Long teamId) {
|
||||
return teamRepository
|
||||
.findById(key(teamId))
|
||||
.flatMap(entity -> parse(entity.getLabelsJson(), "team " + teamId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClassificationLabels save(Long teamId, ClassificationLabels labels, String updatedBy) {
|
||||
TeamLabelsEntity entity = new TeamLabelsEntity();
|
||||
entity.setTeamId(key(teamId));
|
||||
entity.setLabelsJson(objectMapper.writeValueAsString(labels));
|
||||
entity.setUpdatedAt(Instant.now());
|
||||
entity.setUpdatedBy(updatedBy);
|
||||
teamRepository.save(entity);
|
||||
return labels;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteByTeam(Long teamId) {
|
||||
long id = key(teamId);
|
||||
if (!teamRepository.existsById(id)) {
|
||||
return false;
|
||||
}
|
||||
teamRepository.deleteById(id);
|
||||
return true;
|
||||
}
|
||||
|
||||
private Optional<ClassificationLabels> parse(String json, String owner) {
|
||||
try {
|
||||
return Optional.of(objectMapper.readValue(json, ClassificationLabels.class));
|
||||
} catch (JacksonException e) {
|
||||
// A stored label set that no longer parses (corruption / manual DB edit) must not break
|
||||
// classification: drop it so the caller treats the team as having no labels (and skips
|
||||
// classification) rather than surfacing a 500 on every upload.
|
||||
log.warn("Discarding unparseable stored labels for {}: {}", owner, e.getMessage());
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
/** Map the nullable team id onto the entity's non-null key (sentinel for the unteamed case). */
|
||||
private static long key(Long teamId) {
|
||||
return teamId == null ? TeamLabelsEntity.NO_TEAM : teamId;
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Table;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* JPA row for a team's classification labels — one row per team. The label set lives as JSON in
|
||||
* {@code labelsJson} (authoritative on read). {@code teamId} is the natural key; the sentinel
|
||||
* {@link #NO_TEAM} stands in for the unteamed (login-disabled / self-hosted single-team) case,
|
||||
* since a primary key can't be null (policies store a nullable {@code team_id}, but this table is
|
||||
* keyed one-per-team). Kept decoupled from the security entities — {@code teamId} is a plain value,
|
||||
* not a foreign key — so classification can be enabled or disabled without touching them.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "classification_labels")
|
||||
@NoArgsConstructor
|
||||
@Getter
|
||||
@Setter
|
||||
public class TeamLabelsEntity implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/** Sentinel key for the unteamed label set (login disabled / no resolvable team). */
|
||||
public static final long NO_TEAM = 0L;
|
||||
|
||||
@Id
|
||||
@Column(name = "team_id")
|
||||
private long teamId;
|
||||
|
||||
@Column(name = "labels_json", columnDefinition = "text")
|
||||
private String labelsJson;
|
||||
|
||||
@Column(name = "updated_at")
|
||||
private Instant updatedAt;
|
||||
|
||||
@Column(name = "updated_by")
|
||||
private String updatedBy;
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
package stirling.software.proprietary.classification.store;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface TeamLabelsRepository extends JpaRepository<TeamLabelsEntity, Long> {}
|
||||
+32
-14
@@ -31,9 +31,10 @@ import stirling.software.common.service.PdfMetadataService;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.proprietary.classification.ClassificationLabelProvider;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.store.ClassificationLabelStore;
|
||||
import stirling.software.proprietary.model.api.ai.AiPageText;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.service.AiEngineClient;
|
||||
import stirling.software.proprietary.service.PdfContentExtractor;
|
||||
|
||||
@@ -45,7 +46,7 @@ import tools.jackson.databind.node.ObjectNode;
|
||||
* Dispatchable tool that classifies a PDF and writes the result into its metadata.
|
||||
*
|
||||
* <p>Runs as a Classification-policy pipeline step: it reads a bounded page window, asks the AI
|
||||
* engine to classify the document against the built-in label set, and stores the engine's JSON
|
||||
* engine to classify the document against the caller's team label set, and stores the engine's JSON
|
||||
* answer — minus the transport-only {@code outcome} field — in the custom Info-dictionary key
|
||||
* {@link PdfMetadataService#CLASSIFICATION_KEY}. Returns the labelled PDF. Not intended for direct
|
||||
* client use.
|
||||
@@ -71,9 +72,13 @@ public class ClassifyLabelController {
|
||||
private final UserServiceInterface userService;
|
||||
|
||||
/**
|
||||
* The fixed, built-in vocabulary shared by everyone — see {@link ClassificationLabelProvider}.
|
||||
* Present only when the policy subsystem is enabled ({@code policies.enabled}); the store and
|
||||
* team authority are gated on it. Null otherwise, in which case there are no team labels to
|
||||
* classify against and the document is passed through unlabelled.
|
||||
*/
|
||||
private final ClassificationLabelProvider labelProvider;
|
||||
private final ClassificationLabelStore labelStore;
|
||||
|
||||
private final PolicyManagementAuthority policyManagementAuthority;
|
||||
|
||||
public ClassifyLabelController(
|
||||
CustomPDFDocumentFactory pdfDocumentFactory,
|
||||
@@ -82,16 +87,18 @@ public class ClassifyLabelController {
|
||||
PdfMetadataService pdfMetadataService,
|
||||
AiEngineClient aiEngineClient,
|
||||
ObjectMapper objectMapper,
|
||||
ClassificationLabelProvider labelProvider,
|
||||
@Autowired(required = false) UserServiceInterface userService) {
|
||||
@Autowired(required = false) UserServiceInterface userService,
|
||||
@Autowired(required = false) ClassificationLabelStore labelStore,
|
||||
@Autowired(required = false) PolicyManagementAuthority policyManagementAuthority) {
|
||||
this.pdfDocumentFactory = pdfDocumentFactory;
|
||||
this.tempFileManager = tempFileManager;
|
||||
this.pdfContentExtractor = pdfContentExtractor;
|
||||
this.pdfMetadataService = pdfMetadataService;
|
||||
this.aiEngineClient = aiEngineClient;
|
||||
this.objectMapper = objectMapper;
|
||||
this.labelProvider = labelProvider;
|
||||
this.userService = userService;
|
||||
this.labelStore = labelStore;
|
||||
this.policyManagementAuthority = policyManagementAuthority;
|
||||
}
|
||||
|
||||
@PostMapping(value = "/classify-and-label", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@@ -109,9 +116,9 @@ public class ClassifyLabelController {
|
||||
|
||||
List<EngineLabel> allowed = resolveAllowedLabels();
|
||||
if (allowed.isEmpty()) {
|
||||
// No vocabulary to classify against: pass the file through unlabelled rather than
|
||||
// ask the engine to classify against nothing.
|
||||
log.debug("[classify-and-label] {} has no labels; skipping", fileName);
|
||||
// No vocabulary to classify against (the team stored no labels): pass the file
|
||||
// through unlabelled rather than ask the engine to classify against nothing.
|
||||
log.debug("[classify-and-label] {} has no team labels; skipping", fileName);
|
||||
return WebResponseUtils.pdfDocToWebResponse(document, fileName, tempFileManager);
|
||||
}
|
||||
|
||||
@@ -168,13 +175,24 @@ public class ClassifyLabelController {
|
||||
}
|
||||
|
||||
/**
|
||||
* The built-in vocabulary as {@code {id, name}} pairs, de-duplicated by id. The engine shows
|
||||
* the model the names and returns the ids (icons are presentational and never sent). The engine
|
||||
* holds no default vocabulary of its own, so this bundled set is the only source.
|
||||
* The allowed labels for the caller's team as {@code {id, name}} pairs, de-duplicated by id.
|
||||
* The engine shows the model the names and returns the ids (icons are presentational and never
|
||||
* sent). Returns an empty list — the caller then skips classification — when the policy
|
||||
* subsystem is disabled (no store) or the team has no stored labels. The engine holds no
|
||||
* default vocabulary of its own, so a team's stored labels are the only source.
|
||||
*/
|
||||
private List<EngineLabel> resolveAllowedLabels() {
|
||||
if (labelStore == null) {
|
||||
return List.of();
|
||||
}
|
||||
Long teamId =
|
||||
policyManagementAuthority == null
|
||||
? null
|
||||
: policyManagementAuthority.currentUserTeamId();
|
||||
|
||||
Map<String, EngineLabel> byId = new LinkedHashMap<>();
|
||||
collectLabels(labelProvider.labels(), byId);
|
||||
labelStore.findByTeam(teamId).ifPresent(labels -> collectLabels(labels.labels(), byId));
|
||||
|
||||
return List.copyOf(byId.values());
|
||||
}
|
||||
|
||||
|
||||
+17
-38
@@ -8,14 +8,12 @@ import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
@@ -26,24 +24,18 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ProcessExecutor;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.proprietary.model.api.ai.create.AiDocument;
|
||||
import stirling.software.proprietary.service.AiDocumentHtmlRenderer;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Dispatchable tool that converts an AI-generated document model to a PDF via WeasyPrint.
|
||||
* Dispatchable tool that converts an AI-generated HTML string to a PDF via WeasyPrint.
|
||||
*
|
||||
* <p>Called by {@link stirling.software.proprietary.service.AiWorkflowService} when the engine
|
||||
* emits a {@code CREATE_PDF_FROM_HTML_AGENT} plan step. The engine supplies the document as
|
||||
* structured fields; the HTML is built here from a fixed template.
|
||||
* emits a {@code CREATE_PDF_FROM_HTML_AGENT} plan step. The HTML comes from a trusted Jinja
|
||||
* template so sanitization is intentionally skipped.
|
||||
*/
|
||||
@Slf4j
|
||||
@Hidden
|
||||
@@ -56,9 +48,6 @@ public class CreatePdfAgentController {
|
||||
private final TempFileManager tempFileManager;
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final RuntimePathConfig runtimePathConfig;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AiDocumentHtmlRenderer htmlRenderer;
|
||||
|
||||
/**
|
||||
* Returns true only when WeasyPrint is definitively unavailable — either the binary could not
|
||||
@@ -85,42 +74,32 @@ public class CreatePdfAgentController {
|
||||
value = "/create-pdf-from-html-agent",
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Convert an AI-generated document to a PDF",
|
||||
summary = "Convert AI-generated HTML to a PDF",
|
||||
description =
|
||||
"Accepts a structured document as a JSON parameter and returns a PDF. This"
|
||||
+ " endpoint is dispatched by the AI workflow orchestrator as a plan"
|
||||
+ " step; it is not intended for direct client use.")
|
||||
public ResponseEntity<Resource> createPdf(
|
||||
@RequestParam("document") String document, @RequestParam("filename") String filename)
|
||||
"Accepts an HTML document as a plain-text parameter and returns a PDF."
|
||||
+ " This endpoint is dispatched by the AI workflow orchestrator as a"
|
||||
+ " plan step; it is not intended for direct client use.")
|
||||
public ResponseEntity<Resource> createPdfFromHtml(
|
||||
@RequestParam("htmlContent") String htmlContent,
|
||||
@RequestParam("filename") String filename)
|
||||
throws Exception {
|
||||
|
||||
if (!applicationProperties.getAiEngine().isEnabled()) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
|
||||
}
|
||||
|
||||
AiDocument model;
|
||||
try {
|
||||
model = objectMapper.readValue(document, AiDocument.class);
|
||||
} catch (JacksonException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
String html = htmlRenderer.render(model);
|
||||
|
||||
log.info(
|
||||
"[create-pdf-agent] converting document to PDF via WeasyPrint — html_bytes={}",
|
||||
html.length());
|
||||
"[create-pdf-agent] converting HTML to PDF via WeasyPrint — html_bytes={}",
|
||||
htmlContent.length());
|
||||
|
||||
try (TempFile htmlFile = tempFileManager.createManagedTempFile(".html");
|
||||
TempFile pdfFile = tempFileManager.createManagedTempFile(".pdf")) {
|
||||
|
||||
Files.writeString(htmlFile.getPath(), html, StandardCharsets.UTF_8);
|
||||
Files.writeString(htmlFile.getPath(), htmlContent, StandardCharsets.UTF_8);
|
||||
|
||||
List<String> command = new ArrayList<>();
|
||||
command.add(runtimePathConfig.getWeasyPrintPath());
|
||||
command.add("-e");
|
||||
command.add("utf-8");
|
||||
command.add("-v");
|
||||
// SSRF: the HTML is self-contained and the engine validates style colours, so no
|
||||
// external url() reaches WeasyPrint. For full isolation, run it network-isolated.
|
||||
command.add(htmlFile.getAbsolutePath());
|
||||
command.add(pdfFile.getAbsolutePath());
|
||||
|
||||
@@ -147,8 +126,8 @@ public class CreatePdfAgentController {
|
||||
// avoids materialising the whole document as a byte[] twice (read-all + re-serialise),
|
||||
// which matters for large generated documents.
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try (PDDocument pdDocument = pdfDocumentFactory.load(pdfFile.getPath())) {
|
||||
pdDocument.save(tempOut.getPath().toFile());
|
||||
try (PDDocument document = pdfDocumentFactory.load(pdfFile.getPath())) {
|
||||
document.save(tempOut.getPath().toFile());
|
||||
} catch (Exception e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package stirling.software.proprietary.formdetection.catalog;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/** Loads the curated Auto Form Detection model catalog from a bundled JSON resource. */
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ModelCatalogService {
|
||||
|
||||
private static final String CATALOG_RESOURCE = "formdetection/model-catalog.json";
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private volatile List<ModelCatalogEntry> entries = List.of();
|
||||
private volatile Map<String, ModelCatalogEntry> byId = Map.of();
|
||||
|
||||
@PostConstruct
|
||||
void load() {
|
||||
try (InputStream is = new ClassPathResource(CATALOG_RESOURCE).getInputStream()) {
|
||||
List<ModelCatalogEntry> loaded =
|
||||
objectMapper.readValue(is, new TypeReference<List<ModelCatalogEntry>>() {});
|
||||
Map<String, ModelCatalogEntry> map = new LinkedHashMap<>();
|
||||
for (ModelCatalogEntry entry : loaded) {
|
||||
if (entry.getId() != null && !entry.getId().isBlank()) {
|
||||
map.put(entry.getId(), entry);
|
||||
}
|
||||
}
|
||||
this.entries = List.copyOf(map.values());
|
||||
this.byId = Map.copyOf(map);
|
||||
log.info("Loaded {} Auto Form Detection model catalog entries", entries.size());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to load Auto Form Detection model catalog from {}",
|
||||
CATALOG_RESOURCE,
|
||||
e);
|
||||
}
|
||||
}
|
||||
|
||||
public List<ModelCatalogEntry> getAll() {
|
||||
return entries;
|
||||
}
|
||||
|
||||
public Optional<ModelCatalogEntry> getById(String id) {
|
||||
return Optional.ofNullable(id == null ? null : byId.get(id));
|
||||
}
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
package stirling.software.proprietary.formdetection.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.FormUtils.NewFormFieldDefinition;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.proprietary.formdetection.inference.OnnxFormDetector;
|
||||
import stirling.software.proprietary.formdetection.inference.Yolo;
|
||||
import stirling.software.proprietary.formdetection.model.DetectedField;
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
import stirling.software.proprietary.formdetection.render.CoordinateMapper;
|
||||
import stirling.software.proprietary.formdetection.render.PageRasterizer;
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
/**
|
||||
* Server-side detection endpoint. Gated behind the {@code form-detection} endpoint key, which is
|
||||
* disabled until a model is installed (so the tool tile is greyed in the UI). Returns the shared
|
||||
* detection schema, or - when {@code applyToPdf=true} - the AcroForm-applied PDF.
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/ai/form-detection")
|
||||
@ConditionalOnClass(name = "ai.onnxruntime.OrtEnvironment")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Auto Form Detection")
|
||||
public class FormDetectionController {
|
||||
|
||||
private final FormDetectionModelManager manager;
|
||||
private final OnnxFormDetector detector;
|
||||
private final PageRasterizer rasterizer;
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
@PostMapping(value = "/detect", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Detect form fields with the installed AI model",
|
||||
description =
|
||||
"Runs the installed ONNX model over each page and returns detected fields in"
|
||||
+ " PDF points. With applyToPdf=true, returns the fillable PDF instead.")
|
||||
public ResponseEntity<?> detect(
|
||||
@RequestParam("file") MultipartFile file,
|
||||
@RequestParam(value = "confThreshold", required = false) Float confThreshold,
|
||||
@RequestParam(value = "applyToPdf", required = false, defaultValue = "false")
|
||||
boolean applyToPdf)
|
||||
throws IOException {
|
||||
|
||||
if (!manager.isReady()) {
|
||||
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
||||
.body(
|
||||
Map.of(
|
||||
"reason",
|
||||
"DEPENDENCY",
|
||||
"message",
|
||||
"AI form-detection model is not installed"));
|
||||
}
|
||||
ModelCatalogEntry spec = manager.getActiveEntry().orElse(null);
|
||||
if (spec == null) {
|
||||
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
||||
.body(
|
||||
Map.of(
|
||||
"reason",
|
||||
"DEPENDENCY",
|
||||
"message",
|
||||
"Active model spec unavailable"));
|
||||
}
|
||||
float score = confThreshold != null ? confThreshold : spec.getScoreThreshold();
|
||||
byte[] pdfBytes = file.getBytes();
|
||||
|
||||
List<DetectedField> detections = new ArrayList<>();
|
||||
try {
|
||||
for (PageRasterizer.RasterPage page :
|
||||
rasterizer.rasterize(pdfBytes, spec.getInputSize())) {
|
||||
Yolo.Preprocessed pre =
|
||||
Yolo.preprocess(page.rgba(), page.widthPx(), page.heightPx(), spec);
|
||||
Yolo.RawOutput out = detector.infer(pre.chw(), spec.getInputSize());
|
||||
for (Yolo.Detection d : Yolo.decode(out, spec, pre, score)) {
|
||||
DetectedField.RectPt rect = CoordinateMapper.toPdfPoints(d, page);
|
||||
detections.add(
|
||||
new DetectedField(
|
||||
fieldType(spec, d.classId()),
|
||||
page.pageIndex(),
|
||||
rect,
|
||||
d.score()));
|
||||
}
|
||||
}
|
||||
} catch (IllegalStateException e) {
|
||||
// e.g. ONNX Runtime native unavailable for this OS/arch - report unavailable cleanly
|
||||
// rather than a 500. Cannot happen on a normally-built jar (all platforms bundled), but
|
||||
// keeps a slimmed/mis-targeted build from erroring.
|
||||
log.warn("Auto Form Detection inference unavailable: {}", e.getMessage());
|
||||
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
||||
.body(Map.of("reason", "DEPENDENCY", "message", e.getMessage()));
|
||||
}
|
||||
|
||||
if (applyToPdf) {
|
||||
try (PDDocument document = pdfDocumentFactory.load(file)) {
|
||||
FormUtils.repairMissingWidgetPageReferences(document);
|
||||
List<NewFormFieldDefinition> defs = new ArrayList<>();
|
||||
for (DetectedField f : detections) {
|
||||
defs.add(toDefinition(f));
|
||||
}
|
||||
FormUtils.addFields(document, defs);
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document, baseName(file) + ".pdf", tempFileManager);
|
||||
}
|
||||
}
|
||||
return ResponseEntity.ok(new DetectResponse(detections));
|
||||
}
|
||||
|
||||
private static String fieldType(ModelCatalogEntry spec, int classId) {
|
||||
List<String> types = spec.getClassFieldTypes();
|
||||
if (types != null && classId >= 0 && classId < types.size()) {
|
||||
return types.get(classId);
|
||||
}
|
||||
return "text";
|
||||
}
|
||||
|
||||
private static NewFormFieldDefinition toDefinition(DetectedField f) {
|
||||
DetectedField.RectPt r = f.rectInPdfPoints();
|
||||
return new NewFormFieldDefinition(
|
||||
null,
|
||||
null,
|
||||
f.type(),
|
||||
f.page(),
|
||||
(float) r.x(),
|
||||
(float) r.y(),
|
||||
(float) r.w(),
|
||||
(float) r.h(),
|
||||
Boolean.FALSE,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
}
|
||||
|
||||
private static String baseName(MultipartFile file) {
|
||||
String original = Filenames.toSimpleFileName(file.getOriginalFilename());
|
||||
if (original == null || original.isBlank()) {
|
||||
original = "document";
|
||||
}
|
||||
String stem =
|
||||
original.toLowerCase().endsWith(".pdf")
|
||||
? original.substring(0, original.length() - 4)
|
||||
: original;
|
||||
return stem + "_form";
|
||||
}
|
||||
|
||||
/** Shared JSON response (mirrors the browser pipeline output). */
|
||||
public record DetectResponse(List<DetectedField> detections) {}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
package stirling.software.proprietary.formdetection.controller;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelStatusResponse;
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
/**
|
||||
* Admin-managed lifecycle for the Auto Form Detection model. Lives under the never-gated {@code
|
||||
* form-detection-model} endpoint key so install/status stay reachable while the feature itself (the
|
||||
* {@code form-detection} detect endpoint) is disabled until a model is ready.
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/ai/form-detection-model")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Auto Form Detection")
|
||||
public class FormDetectionModelController {
|
||||
|
||||
private final FormDetectionModelManager manager;
|
||||
|
||||
@GetMapping("/status")
|
||||
@Operation(summary = "Auto Form Detection model status, progress and catalog")
|
||||
public ResponseEntity<ModelStatusResponse> status() {
|
||||
return ResponseEntity.ok(manager.status());
|
||||
}
|
||||
|
||||
@PostMapping("/install")
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@Operation(summary = "Install (download + checksum-verify) a catalog model")
|
||||
public ResponseEntity<ModelStatusResponse> install(@RequestBody InstallRequest request) {
|
||||
if (request == null || StringUtils.isBlank(request.getModelId())) {
|
||||
ModelStatusResponse s = manager.status();
|
||||
s.setError("modelId is required");
|
||||
return ResponseEntity.badRequest().body(s);
|
||||
}
|
||||
try {
|
||||
manager.startInstall(request.getModelId());
|
||||
return ResponseEntity.accepted().body(manager.status());
|
||||
} catch (IllegalStateException e) {
|
||||
ModelStatusResponse s = manager.status();
|
||||
s.setError(e.getMessage());
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT).body(s);
|
||||
} catch (IllegalArgumentException e) {
|
||||
ModelStatusResponse s = manager.status();
|
||||
s.setError(e.getMessage());
|
||||
return ResponseEntity.badRequest().body(s);
|
||||
}
|
||||
}
|
||||
|
||||
@DeleteMapping
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@Operation(summary = "Uninstall a model")
|
||||
public ResponseEntity<ModelStatusResponse> delete(
|
||||
@RequestParam(name = "modelId", required = false) String modelId) {
|
||||
try {
|
||||
manager.deleteModel(modelId);
|
||||
return ResponseEntity.ok(manager.status());
|
||||
} catch (IllegalStateException e) {
|
||||
ModelStatusResponse s = manager.status();
|
||||
s.setError(e.getMessage());
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT).body(s);
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("/config")
|
||||
@PreAuthorize("hasRole('ADMIN')")
|
||||
@Operation(
|
||||
summary = "Update the feature on/off switch and execution mode (auto/browser/server)")
|
||||
public ResponseEntity<ModelStatusResponse> config(@RequestBody ConfigRequest request) {
|
||||
if (request == null) {
|
||||
return ResponseEntity.badRequest().body(manager.status());
|
||||
}
|
||||
try {
|
||||
if (request.getEnabled() != null) {
|
||||
manager.setEnabled(request.getEnabled());
|
||||
}
|
||||
if (StringUtils.isNotBlank(request.getExecutionMode())) {
|
||||
manager.setExecutionMode(request.getExecutionMode());
|
||||
}
|
||||
return ResponseEntity.ok(manager.status());
|
||||
} catch (IllegalArgumentException e) {
|
||||
ModelStatusResponse s = manager.status();
|
||||
s.setError(e.getMessage());
|
||||
return ResponseEntity.badRequest().body(s);
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class ConfigRequest {
|
||||
/** Master on/off; {@code null} leaves it unchanged. */
|
||||
private Boolean enabled;
|
||||
|
||||
/** auto|browser|server; blank/null leaves it unchanged. */
|
||||
private String executionMode;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class InstallRequest {
|
||||
private String modelId;
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package stirling.software.proprietary.formdetection.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.support.ResourceRegion;
|
||||
import org.springframework.http.CacheControl;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpRange;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
/**
|
||||
* Streams the installed .onnx to browsers for in-browser inference. Supports resumable range
|
||||
* requests with a stable (content-addressed) ETag and a public Cache-Control so each browser
|
||||
* fetches the ~200MB model at most once. The file is streamed from disk by Spring's resource
|
||||
* converters - never buffered into heap.
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/ai/form-detection-model")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Auto Form Detection")
|
||||
public class FormDetectionModelServeController {
|
||||
|
||||
private final FormDetectionModelManager manager;
|
||||
|
||||
@GetMapping("/file")
|
||||
@Operation(summary = "Stream the installed model (.onnx); supports HTTP range requests")
|
||||
public ResponseEntity<Object> serveModel(@RequestHeader HttpHeaders headers)
|
||||
throws IOException {
|
||||
Optional<Path> active = manager.getActiveModelFile();
|
||||
if (active.isEmpty()) {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
Path path = active.get();
|
||||
Resource resource = new FileSystemResource(path);
|
||||
long length = resource.contentLength();
|
||||
String etag =
|
||||
"\""
|
||||
+ manager.getActiveEtag()
|
||||
.orElseGet(() -> length + "-" + path.toFile().lastModified())
|
||||
+ "\"";
|
||||
// cachePublic + maxAge override the interceptor's blanket "no-store" on /api responses.
|
||||
CacheControl cacheControl = CacheControl.maxAge(Duration.ofDays(30)).cachePublic();
|
||||
|
||||
List<HttpRange> ranges;
|
||||
try {
|
||||
ranges = headers.getRange();
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return ResponseEntity.status(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
|
||||
.header(HttpHeaders.CONTENT_RANGE, "bytes */" + length)
|
||||
.build();
|
||||
}
|
||||
|
||||
if (ranges.isEmpty()) {
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.ACCEPT_RANGES, "bytes")
|
||||
.eTag(etag)
|
||||
.cacheControl(cacheControl)
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.contentLength(length)
|
||||
.body(resource);
|
||||
}
|
||||
|
||||
// Serve a single region (the resumable-download case). Returning a bare List would lose its
|
||||
// generic element type when the method returns ResponseEntity<Object>, leaving the region
|
||||
// converter unable to match it.
|
||||
HttpRange range = ranges.get(0);
|
||||
long start = range.getRangeStart(length);
|
||||
long count = Math.min(range.getRangeEnd(length) - start + 1, length - start);
|
||||
ResourceRegion region = new ResourceRegion(resource, start, count);
|
||||
// No explicit content type: the region converter derives it. Presetting octet-stream makes
|
||||
// the ResourceRegion converter's content-negotiation reject the body.
|
||||
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT)
|
||||
.header(HttpHeaders.ACCEPT_RANGES, "bytes")
|
||||
.eTag(etag)
|
||||
.cacheControl(cacheControl)
|
||||
.body(region);
|
||||
}
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package stirling.software.proprietary.formdetection.inference;
|
||||
|
||||
import java.nio.FloatBuffer;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
import ai.onnxruntime.OnnxTensor;
|
||||
import ai.onnxruntime.OnnxValue;
|
||||
import ai.onnxruntime.OrtEnvironment;
|
||||
import ai.onnxruntime.OrtException;
|
||||
import ai.onnxruntime.OrtSession;
|
||||
|
||||
/**
|
||||
* Holds the ONNX Runtime session for the active model. Lazily (re)loads when the active model
|
||||
* changes, guards session swaps with a read/write lock, and bounds concurrent inferences to limit
|
||||
* memory. The session input is NCHW float32 {@code [1,3,N,N]}; the raw output is returned as-is for
|
||||
* {@link Yolo#decode} to interpret per the model spec.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@ConditionalOnClass(name = "ai.onnxruntime.OrtEnvironment")
|
||||
@RequiredArgsConstructor
|
||||
public class OnnxFormDetector {
|
||||
|
||||
private final FormDetectionModelManager manager;
|
||||
|
||||
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
|
||||
private final Semaphore concurrency =
|
||||
new Semaphore(Math.max(1, Runtime.getRuntime().availableProcessors() / 2));
|
||||
|
||||
private volatile OrtSession session;
|
||||
private volatile String loadedModelId;
|
||||
private volatile String inputName;
|
||||
|
||||
public Yolo.RawOutput infer(float[] chw, int inputSize) {
|
||||
ensureLoaded();
|
||||
concurrency.acquireUninterruptibly();
|
||||
lock.readLock().lock();
|
||||
try {
|
||||
OrtEnvironment env = OrtEnvironment.getEnvironment();
|
||||
long[] shape = {1, 3, inputSize, inputSize};
|
||||
try (OnnxTensor tensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(chw), shape);
|
||||
OrtSession.Result results =
|
||||
session.run(Collections.singletonMap(inputName, tensor))) {
|
||||
OnnxValue value = results.get(0);
|
||||
Object raw = value.getValue();
|
||||
if (!(raw instanceof float[][][] out3) || out3.length == 0) {
|
||||
throw new IllegalStateException(
|
||||
"Unexpected ONNX output type: "
|
||||
+ (raw == null ? "null" : raw.getClass()));
|
||||
}
|
||||
float[][] m = out3[0];
|
||||
int d1 = m.length;
|
||||
int d2 = d1 > 0 ? m[0].length : 0;
|
||||
float[] flat = new float[d1 * d2];
|
||||
for (int i = 0; i < d1; i++) {
|
||||
System.arraycopy(m[i], 0, flat, i * d2, d2);
|
||||
}
|
||||
return new Yolo.RawOutput(flat, d1, d2);
|
||||
}
|
||||
} catch (OrtException e) {
|
||||
throw new IllegalStateException("ONNX inference failed: " + e.getMessage(), e);
|
||||
} finally {
|
||||
lock.readLock().unlock();
|
||||
concurrency.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Force the next inference to reload from disk (called after install/uninstall). */
|
||||
public void unload() {
|
||||
lock.writeLock().lock();
|
||||
try {
|
||||
closeSession();
|
||||
loadedModelId = null;
|
||||
inputName = null;
|
||||
} finally {
|
||||
lock.writeLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureLoaded() {
|
||||
String activeId = manager.getActiveEntry().map(ModelCatalogEntry::getId).orElse(null);
|
||||
if (activeId == null) {
|
||||
throw new IllegalStateException("No Auto Form Detection model installed");
|
||||
}
|
||||
if (activeId.equals(loadedModelId) && session != null) {
|
||||
return;
|
||||
}
|
||||
lock.writeLock().lock();
|
||||
try {
|
||||
if (activeId.equals(loadedModelId) && session != null) {
|
||||
return;
|
||||
}
|
||||
Path file =
|
||||
manager.getActiveModelFile()
|
||||
.orElseThrow(
|
||||
() -> new IllegalStateException("Active model file missing"));
|
||||
try {
|
||||
OrtEnvironment env = OrtEnvironment.getEnvironment();
|
||||
OrtSession.SessionOptions opts = new OrtSession.SessionOptions();
|
||||
try {
|
||||
opts.setIntraOpNumThreads(
|
||||
Math.max(1, Runtime.getRuntime().availableProcessors() / 2));
|
||||
} catch (OrtException ignored) {
|
||||
// best-effort tuning
|
||||
}
|
||||
closeSession();
|
||||
session = env.createSession(file.toString(), opts);
|
||||
inputName = session.getInputNames().iterator().next();
|
||||
loadedModelId = activeId;
|
||||
log.info("Loaded ONNX session for Auto Form Detection model '{}'", activeId);
|
||||
} catch (OrtException | RuntimeException | LinkageError e) {
|
||||
// Native library missing/incompatible for this OS+arch (e.g. a Linux-slimmed jar
|
||||
// run on Windows), or a model load failure. Degrade gracefully instead of letting
|
||||
// an UnsatisfiedLinkError escape - the detect endpoint reports unavailable and the
|
||||
// server keeps running.
|
||||
throw new IllegalStateException(
|
||||
"ONNX Runtime is unavailable on this platform/build: " + e.getMessage(), e);
|
||||
}
|
||||
} finally {
|
||||
lock.writeLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void close() {
|
||||
unload();
|
||||
}
|
||||
|
||||
private void closeSession() {
|
||||
if (session != null) {
|
||||
try {
|
||||
session.close();
|
||||
} catch (Exception ignored) {
|
||||
// ignore
|
||||
}
|
||||
session = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+252
@@ -0,0 +1,252 @@
|
||||
package stirling.software.proprietary.formdetection.inference;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
|
||||
/**
|
||||
* Pure pre/post-processing for a YOLO-style detector, driven entirely by the {@link
|
||||
* ModelCatalogEntry} spec. The browser pipeline mirrors this exactly so both inference paths agree.
|
||||
*
|
||||
* <p>Coordinate spaces: {@code preprocess} maps the source bitmap into the model's NxN input;
|
||||
* {@code decode} reads raw model output (boxes in input-pixel space), thresholds, runs NMS, and
|
||||
* un-projects boxes back to the original bitmap-pixel space (top-left origin). Mapping to PDF
|
||||
* points is done separately by {@code CoordinateMapper}.
|
||||
*/
|
||||
@Slf4j
|
||||
public final class Yolo {
|
||||
|
||||
private Yolo() {}
|
||||
|
||||
/** Normalised model input plus the transform needed to invert it. */
|
||||
public record Preprocessed(
|
||||
float[] chw,
|
||||
int inputSize,
|
||||
float scaleX,
|
||||
float scaleY,
|
||||
int padX,
|
||||
int padY,
|
||||
int srcW,
|
||||
int srcH) {}
|
||||
|
||||
/** Raw model output flattened to {@code data[i*d2 + j]} with dims {@code d1 x d2}. */
|
||||
public record RawOutput(float[] data, int d1, int d2) {}
|
||||
|
||||
/** A detection in original bitmap-pixel space, top-left origin. */
|
||||
public record Detection(int classId, float score, float x, float y, float w, float h) {}
|
||||
|
||||
/** Letterbox/stretch-resize, normalise and lay out as NCHW float32. */
|
||||
public static Preprocessed preprocess(byte[] rgba, int srcW, int srcH, ModelCatalogEntry spec) {
|
||||
int n = spec.getInputSize();
|
||||
boolean letterbox = !"stretch".equalsIgnoreCase(spec.getResizeMode());
|
||||
|
||||
float scaleX;
|
||||
float scaleY;
|
||||
int padX;
|
||||
int padY;
|
||||
int drawW;
|
||||
int drawH;
|
||||
if (letterbox) {
|
||||
float scale = Math.min((float) n / srcW, (float) n / srcH);
|
||||
drawW = Math.max(1, Math.round(srcW * scale));
|
||||
drawH = Math.max(1, Math.round(srcH * scale));
|
||||
padX = (n - drawW) / 2;
|
||||
padY = (n - drawH) / 2;
|
||||
scaleX = scale;
|
||||
scaleY = scale;
|
||||
} else {
|
||||
drawW = n;
|
||||
drawH = n;
|
||||
padX = 0;
|
||||
padY = 0;
|
||||
scaleX = (float) n / srcW;
|
||||
scaleY = (float) n / srcH;
|
||||
}
|
||||
|
||||
int[] pad = spec.getPadColor();
|
||||
Color fill =
|
||||
new Color(
|
||||
clampByte(pad != null && pad.length > 0 ? pad[0] : 114),
|
||||
clampByte(pad != null && pad.length > 1 ? pad[1] : 114),
|
||||
clampByte(pad != null && pad.length > 2 ? pad[2] : 114));
|
||||
|
||||
BufferedImage canvas = new BufferedImage(n, n, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = canvas.createGraphics();
|
||||
try {
|
||||
g.setRenderingHint(
|
||||
RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
|
||||
g.setColor(fill);
|
||||
g.fillRect(0, 0, n, n);
|
||||
g.drawImage(rgbaToImage(rgba, srcW, srcH), padX, padY, drawW, drawH, null);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
|
||||
boolean bgr = "bgr".equalsIgnoreCase(spec.getChannelOrder());
|
||||
float[] mean = orZeros(spec.getNormMean());
|
||||
float[] std = orOnes(spec.getNormStd());
|
||||
int plane = n * n;
|
||||
float[] chw = new float[3 * plane];
|
||||
int[] px = canvas.getRGB(0, 0, n, n, null, 0, n);
|
||||
for (int i = 0; i < plane; i++) {
|
||||
int rgb = px[i];
|
||||
float r = ((rgb >> 16) & 0xFF) / 255f;
|
||||
float gg = ((rgb >> 8) & 0xFF) / 255f;
|
||||
float b = (rgb & 0xFF) / 255f;
|
||||
float c0 = bgr ? b : r;
|
||||
float c1 = gg;
|
||||
float c2 = bgr ? r : b;
|
||||
chw[i] = (c0 - mean[0]) / std[0];
|
||||
chw[plane + i] = (c1 - mean[1]) / std[1];
|
||||
chw[2 * plane + i] = (c2 - mean[2]) / std[2];
|
||||
}
|
||||
return new Preprocessed(chw, n, scaleX, scaleY, padX, padY, srcW, srcH);
|
||||
}
|
||||
|
||||
/** Decode raw output, threshold, NMS, and un-project to original bitmap pixels. */
|
||||
public static List<Detection> decode(
|
||||
RawOutput out, ModelCatalogEntry spec, Preprocessed pre, float scoreThreshold) {
|
||||
int numClasses = spec.getClassNames() == null ? 0 : spec.getClassNames().size();
|
||||
if (numClasses == 0) {
|
||||
return List.of();
|
||||
}
|
||||
boolean obj = spec.isHasObjectness();
|
||||
boolean ncFirst = !"anchors_first".equalsIgnoreCase(spec.getOutputLayout());
|
||||
int channels = ncFirst ? out.d1() : out.d2();
|
||||
int anchors = ncFirst ? out.d2() : out.d1();
|
||||
int expected = 4 + (obj ? 1 : 0) + numClasses;
|
||||
if (channels < expected) {
|
||||
log.warn(
|
||||
"ONNX output channel count {} < expected {} (4 + obj + {} classes); skipping",
|
||||
channels,
|
||||
expected,
|
||||
numClasses);
|
||||
return List.of();
|
||||
}
|
||||
int classOffset = 4 + (obj ? 1 : 0);
|
||||
float[] data = out.data();
|
||||
|
||||
List<Detection> dets = new ArrayList<>();
|
||||
for (int a = 0; a < anchors; a++) {
|
||||
float objScore = obj ? at(data, ncFirst, anchors, channels, 4, a) : 1f;
|
||||
int bestClass = -1;
|
||||
float bestScore = 0f;
|
||||
for (int c = 0; c < numClasses; c++) {
|
||||
float s = at(data, ncFirst, anchors, channels, classOffset + c, a) * objScore;
|
||||
if (s > bestScore) {
|
||||
bestScore = s;
|
||||
bestClass = c;
|
||||
}
|
||||
}
|
||||
if (bestClass < 0 || bestScore < scoreThreshold) {
|
||||
continue;
|
||||
}
|
||||
float cx = at(data, ncFirst, anchors, channels, 0, a);
|
||||
float cy = at(data, ncFirst, anchors, channels, 1, a);
|
||||
float w = at(data, ncFirst, anchors, channels, 2, a);
|
||||
float h = at(data, ncFirst, anchors, channels, 3, a);
|
||||
float x1 = cx - w / 2f;
|
||||
float y1 = cy - h / 2f;
|
||||
float ox = (x1 - pre.padX()) / pre.scaleX();
|
||||
float oy = (y1 - pre.padY()) / pre.scaleY();
|
||||
float ow = w / pre.scaleX();
|
||||
float oh = h / pre.scaleY();
|
||||
// clamp to the source bitmap
|
||||
float cxl = Math.max(0, Math.min(ox, pre.srcW()));
|
||||
float cyl = Math.max(0, Math.min(oy, pre.srcH()));
|
||||
ow = Math.max(0, Math.min(ow, pre.srcW() - cxl));
|
||||
oh = Math.max(0, Math.min(oh, pre.srcH() - cyl));
|
||||
if (ow <= 0 || oh <= 0) {
|
||||
continue;
|
||||
}
|
||||
dets.add(new Detection(bestClass, bestScore, cxl, cyl, ow, oh));
|
||||
}
|
||||
return nms(dets, spec.getNms(), spec.getIou());
|
||||
}
|
||||
|
||||
private static float at(
|
||||
float[] data, boolean ncFirst, int anchors, int channels, int c, int a) {
|
||||
return ncFirst ? data[c * anchors + a] : data[a * channels + c];
|
||||
}
|
||||
|
||||
private static List<Detection> nms(List<Detection> dets, String mode, float iouThreshold) {
|
||||
if (dets.size() < 2 || "none".equalsIgnoreCase(mode)) {
|
||||
return dets;
|
||||
}
|
||||
boolean classAgnostic = mode != null && mode.toLowerCase().contains("agnostic");
|
||||
List<Detection> sorted = new ArrayList<>(dets);
|
||||
sorted.sort((x, y) -> Float.compare(y.score(), x.score()));
|
||||
boolean[] removed = new boolean[sorted.size()];
|
||||
List<Detection> keep = new ArrayList<>();
|
||||
for (int i = 0; i < sorted.size(); i++) {
|
||||
if (removed[i]) {
|
||||
continue;
|
||||
}
|
||||
Detection di = sorted.get(i);
|
||||
keep.add(di);
|
||||
for (int j = i + 1; j < sorted.size(); j++) {
|
||||
if (removed[j]) {
|
||||
continue;
|
||||
}
|
||||
Detection dj = sorted.get(j);
|
||||
if (!classAgnostic && di.classId() != dj.classId()) {
|
||||
continue;
|
||||
}
|
||||
if (iou(di, dj) > iouThreshold) {
|
||||
removed[j] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return keep;
|
||||
}
|
||||
|
||||
private static float iou(Detection a, Detection b) {
|
||||
float ax2 = a.x() + a.w();
|
||||
float ay2 = a.y() + a.h();
|
||||
float bx2 = b.x() + b.w();
|
||||
float by2 = b.y() + b.h();
|
||||
float ix1 = Math.max(a.x(), b.x());
|
||||
float iy1 = Math.max(a.y(), b.y());
|
||||
float ix2 = Math.min(ax2, bx2);
|
||||
float iy2 = Math.min(ay2, by2);
|
||||
float iw = Math.max(0, ix2 - ix1);
|
||||
float ih = Math.max(0, iy2 - iy1);
|
||||
float inter = iw * ih;
|
||||
float union = a.w() * a.h() + b.w() * b.h() - inter;
|
||||
return union <= 0 ? 0 : inter / union;
|
||||
}
|
||||
|
||||
private static BufferedImage rgbaToImage(byte[] rgba, int w, int h) {
|
||||
BufferedImage img = new BufferedImage(w, h, BufferedImage.TYPE_INT_ARGB);
|
||||
int[] px = new int[w * h];
|
||||
int pixels = Math.min(w * h, rgba.length / 4);
|
||||
for (int i = 0; i < pixels; i++) {
|
||||
int r = rgba[i * 4] & 0xFF;
|
||||
int g = rgba[i * 4 + 1] & 0xFF;
|
||||
int b = rgba[i * 4 + 2] & 0xFF;
|
||||
int a = rgba[i * 4 + 3] & 0xFF;
|
||||
px[i] = (a << 24) | (r << 16) | (g << 8) | b;
|
||||
}
|
||||
img.setRGB(0, 0, w, h, px, 0, w);
|
||||
return img;
|
||||
}
|
||||
|
||||
private static int clampByte(int v) {
|
||||
return Math.max(0, Math.min(255, v));
|
||||
}
|
||||
|
||||
private static float[] orZeros(float[] v) {
|
||||
return v != null && v.length >= 3 ? v : new float[] {0f, 0f, 0f};
|
||||
}
|
||||
|
||||
private static float[] orOnes(float[] v) {
|
||||
return v != null && v.length >= 3 ? v : new float[] {1f, 1f, 1f};
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.proprietary.formdetection.model;
|
||||
|
||||
/**
|
||||
* One detected form field. This is the shared schema returned by both the server detect endpoint
|
||||
* and the in-browser pipeline so the two paths are interchangeable.
|
||||
*
|
||||
* @param type AcroForm field type (text|checkbox|radio|signature)
|
||||
* @param page zero-based page index
|
||||
* @param rectInPdfPoints rectangle in PDF points (bottom-left origin)
|
||||
* @param confidence detection confidence 0-1
|
||||
*/
|
||||
public record DetectedField(String type, int page, RectPt rectInPdfPoints, double confidence) {
|
||||
|
||||
/** Rectangle in PDF points, bottom-left origin (PDF user space). */
|
||||
public record RectPt(double x, double y, double w, double h) {}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.formdetection.model;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/** Lifecycle state of the Auto Form Detection model install. */
|
||||
public enum FormDetectionStatus {
|
||||
NOT_INSTALLED,
|
||||
DOWNLOADING,
|
||||
VERIFYING,
|
||||
READY,
|
||||
FAILED;
|
||||
|
||||
/** Lowercase wire form sent to the frontend, e.g. {@code not_installed}. */
|
||||
public String wire() {
|
||||
return name().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package stirling.software.proprietary.formdetection.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* One installable form-detection model plus the data-driven pre/post-processing spec the inference
|
||||
* pipeline needs. The exact same numeric values are surfaced to the browser (via the model status
|
||||
* endpoint) so the in-browser and server inference paths stay equivalent.
|
||||
*
|
||||
* <p>NOTE: the pipeline-spec defaults below follow common Ultralytics-YOLO conventions. The precise
|
||||
* values for a given model (input size, resize mode, channel order, output layout, NMS, class
|
||||
* indices) MUST be verified against the actual exported {@code .onnx} before that entry's {@code
|
||||
* onnxUrl}/{@code sha256} are populated. An entry with a blank {@code onnxUrl} or {@code sha256} is
|
||||
* not installable, which keeps the distribution shippable without any bundled model.
|
||||
*/
|
||||
@Data
|
||||
public class ModelCatalogEntry {
|
||||
|
||||
// --- Identity / distribution -------------------------------------------------
|
||||
private String id;
|
||||
private String displayName;
|
||||
private String description;
|
||||
private String license;
|
||||
private long sizeBytes;
|
||||
|
||||
/** Direct download URL of the .onnx. Blank = not yet available (install is rejected). */
|
||||
private String onnxUrl;
|
||||
|
||||
/** Lower-hex SHA-256 of the .onnx. Blank = not yet available (install is rejected). */
|
||||
private String sha256;
|
||||
|
||||
// --- Pre-processing (parity-critical, mirrored by the browser) ---------------
|
||||
/** Square model input edge in pixels. */
|
||||
private int inputSize = 1024;
|
||||
|
||||
/** "letterbox" (aspect-preserving pad) or "stretch" (resize to square). */
|
||||
private String resizeMode = "letterbox";
|
||||
|
||||
/** RGB letterbox pad colour. */
|
||||
private int[] padColor = {114, 114, 114};
|
||||
|
||||
/** "rgb" or "bgr" channel order fed to the model. */
|
||||
private String channelOrder = "rgb";
|
||||
|
||||
/**
|
||||
* Per-channel mean subtracted after dividing the raw byte by 255 ({@code (raw/255 -
|
||||
* mean)/std}).
|
||||
*/
|
||||
private float[] normMean = {0f, 0f, 0f};
|
||||
|
||||
/** Per-channel std applied after mean subtraction. */
|
||||
private float[] normStd = {1f, 1f, 1f};
|
||||
|
||||
// --- Post-processing (parity-critical) ---------------------------------------
|
||||
/** "nc_first" => output [1, 4+nc, anchors]; "anchors_first" => [1, anchors, 4+nc]. */
|
||||
private String outputLayout = "nc_first";
|
||||
|
||||
/** True if an objectness score column precedes the class scores (YOLOv5 style). */
|
||||
private boolean hasObjectness = false;
|
||||
|
||||
/** Class index -> label. */
|
||||
private List<String> classNames = List.of("text", "choice", "signature");
|
||||
|
||||
/** Class index -> AcroForm field type (text|checkbox|radio|signature). */
|
||||
private List<String> classFieldTypes = List.of("text", "checkbox", "signature");
|
||||
|
||||
private float scoreThreshold = 0.25f;
|
||||
|
||||
/** "none", "classAgnostic" or "perClass". */
|
||||
private String nms = "perClass";
|
||||
|
||||
private float iou = 0.45f;
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package stirling.software.proprietary.formdetection.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* Snapshot returned by {@code GET /api/v1/ai/form-detection-model/status}. Includes the full
|
||||
* catalog so the browser can read the active model's parity-critical pipeline spec.
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ModelStatusResponse {
|
||||
/** Wire state: not_installed | downloading | verifying | ready | failed. */
|
||||
private String status;
|
||||
|
||||
/** Download progress 0-100 (meaningful while downloading). */
|
||||
private int progress;
|
||||
|
||||
/** Id of the active/usable model, or blank when none. */
|
||||
private String activeModelId;
|
||||
|
||||
/** Model ids that currently have an .onnx file on disk. */
|
||||
private List<String> installed;
|
||||
|
||||
/** Last error message, or null. */
|
||||
private String error;
|
||||
|
||||
/** Whether the model directory is writable (admin install possible). */
|
||||
private boolean writable;
|
||||
|
||||
/** Full curated catalog (identity + pipeline spec). */
|
||||
private List<ModelCatalogEntry> catalog;
|
||||
|
||||
/** Master on/off for the whole feature (admin-controlled). */
|
||||
private boolean enabled;
|
||||
|
||||
/** Where detection runs: auto | browser | server. */
|
||||
private String executionMode;
|
||||
|
||||
/** True if the server-side ONNX engine is bundled in this build (else only browser works). */
|
||||
private boolean serverEngineAvailable;
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package stirling.software.proprietary.formdetection.render;
|
||||
|
||||
import stirling.software.proprietary.formdetection.inference.Yolo;
|
||||
import stirling.software.proprietary.formdetection.model.DetectedField;
|
||||
|
||||
/**
|
||||
* Maps a detection (original bitmap pixels, top-left origin) to PDF points (bottom-left origin),
|
||||
* accounting for the render scale and the top-left vs bottom-left origin flip.
|
||||
*/
|
||||
public final class CoordinateMapper {
|
||||
|
||||
private CoordinateMapper() {}
|
||||
|
||||
public static DetectedField.RectPt toPdfPoints(
|
||||
Yolo.Detection d, PageRasterizer.RasterPage page) {
|
||||
float sx = page.scaleX() > 0 ? page.scaleX() : 1f;
|
||||
float sy = page.scaleY() > 0 ? page.scaleY() : 1f;
|
||||
|
||||
double wPt = d.w() / sx;
|
||||
double hPt = d.h() / sy;
|
||||
double xPt = d.x() / sx;
|
||||
// Flip Y: bitmap origin is top-left, PDF origin is bottom-left.
|
||||
double yPt = page.pageHeightPt() - (d.y() / sy) - hPt;
|
||||
|
||||
xPt = clamp(xPt, 0, page.pageWidthPt());
|
||||
yPt = clamp(yPt, 0, page.pageHeightPt());
|
||||
wPt = clamp(wPt, 0, page.pageWidthPt() - xPt);
|
||||
hPt = clamp(hPt, 0, page.pageHeightPt() - yPt);
|
||||
return new DetectedField.RectPt(xPt, yPt, wPt, hPt);
|
||||
}
|
||||
|
||||
private static double clamp(double v, double lo, double hi) {
|
||||
return v < lo ? lo : Math.min(v, hi);
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package stirling.software.proprietary.formdetection.render;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.model.PageSize;
|
||||
import stirling.software.jpdfium.model.RenderResult;
|
||||
|
||||
/**
|
||||
* Renders PDF pages to RGBA bitmaps via JPDFium (the same PDFium engine the browser pipeline uses,
|
||||
* for closer parity than PDFBox's Java2D renderer). Each page is rendered at a DPI chosen so its
|
||||
* long side is approximately the model input size, minimising any later resampling. The actual
|
||||
* pixels-per-point scale is computed from the rendered dimensions so coordinate mapping does not
|
||||
* depend on how {@code renderAt} interprets its argument.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class PageRasterizer {
|
||||
|
||||
/** A rendered page: RGBA pixels plus the page size (points) and px-per-point scale. */
|
||||
public record RasterPage(
|
||||
int pageIndex,
|
||||
byte[] rgba,
|
||||
int widthPx,
|
||||
int heightPx,
|
||||
float pageWidthPt,
|
||||
float pageHeightPt,
|
||||
float scaleX,
|
||||
float scaleY) {}
|
||||
|
||||
public List<RasterPage> rasterize(byte[] pdfBytes, int inputSize) {
|
||||
List<RasterPage> pages = new ArrayList<>();
|
||||
try (PdfDocument doc = PdfDocument.open(pdfBytes)) {
|
||||
int count = doc.pageCount();
|
||||
for (int i = 0; i < count; i++) {
|
||||
try (PdfPage page = doc.page(i)) {
|
||||
PageSize size = page.size();
|
||||
float maxSide = Math.max(size.width(), size.height());
|
||||
int dpi = maxSide <= 0 ? 150 : Math.round(72f * inputSize / maxSide);
|
||||
dpi = Math.max(36, Math.min(dpi, 300));
|
||||
RenderResult r = page.renderAt(dpi);
|
||||
float scaleX = size.width() > 0 ? r.width() / size.width() : dpi / 72f;
|
||||
float scaleY = size.height() > 0 ? r.height() / size.height() : dpi / 72f;
|
||||
pages.add(
|
||||
new RasterPage(
|
||||
i,
|
||||
r.rgba(),
|
||||
r.width(),
|
||||
r.height(),
|
||||
size.width(),
|
||||
size.height(),
|
||||
scaleX,
|
||||
scaleY));
|
||||
}
|
||||
}
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
}
|
||||
+509
@@ -0,0 +1,509 @@
|
||||
package stirling.software.proprietary.formdetection.service;
|
||||
|
||||
import static java.nio.file.StandardOpenOption.CREATE;
|
||||
import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
|
||||
import static java.nio.file.StandardOpenOption.WRITE;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.SPDF.config.EndpointConfiguration.DisableReason;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.proprietary.formdetection.catalog.ModelCatalogService;
|
||||
import stirling.software.proprietary.formdetection.model.FormDetectionStatus;
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
import stirling.software.proprietary.formdetection.model.ModelStatusResponse;
|
||||
|
||||
/**
|
||||
* Downloads, verifies and tracks the on-demand Auto Form Detection model. Concurrency-safe
|
||||
* (single-flight install), checksum-verified, and atomic-published to a mounted volume so the model
|
||||
* survives container restarts/updates. Mirrors the OCR tessdata admin pattern but adds the lock,
|
||||
* temp-file + atomic rename, and SHA-256 verification the spec requires.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class FormDetectionModelManager {
|
||||
|
||||
/** Endpoint key gated until a model is ready (drives the disabled tool tile in the UI). */
|
||||
public static final String ENDPOINT_KEY = "form-detection";
|
||||
|
||||
private static final Pattern SAFE_ID = Pattern.compile("[a-z0-9][a-z0-9-]{0,63}");
|
||||
private static final Pattern SHA256_HEX = Pattern.compile("[0-9a-f]{64}");
|
||||
|
||||
/**
|
||||
* Whether the server-side ONNX engine is bundled in this build (the onnxruntime jar is only
|
||||
* included via {@code -PbundleOnnxRuntime=true}, e.g. the Docker server image). The frontend
|
||||
* uses this to disable the "Server" execution mode when it isn't available.
|
||||
*/
|
||||
private static final boolean SERVER_ENGINE_AVAILABLE = isOnnxRuntimePresent();
|
||||
|
||||
private static boolean isOnnxRuntimePresent() {
|
||||
try {
|
||||
Class.forName(
|
||||
"ai.onnxruntime.OrtEnvironment",
|
||||
false,
|
||||
FormDetectionModelManager.class.getClassLoader());
|
||||
return true;
|
||||
} catch (Throwable t) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private final RuntimePathConfig runtimePathConfig;
|
||||
private final ModelCatalogService catalog;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final EndpointConfiguration endpointConfiguration;
|
||||
|
||||
private final AtomicBoolean installing = new AtomicBoolean(false);
|
||||
private volatile FormDetectionStatus state = FormDetectionStatus.NOT_INSTALLED;
|
||||
private volatile int progress = 0;
|
||||
private volatile String error = null;
|
||||
private volatile String activeSha = null;
|
||||
|
||||
@PostConstruct
|
||||
void init() {
|
||||
sweepTempFiles();
|
||||
seedPreinstalledModels();
|
||||
Optional<Path> active = getActiveModelFile();
|
||||
if (active.isPresent()) {
|
||||
activeSha = getActiveEntry().map(ModelCatalogEntry::getSha256).orElse(null);
|
||||
state = FormDetectionStatus.READY;
|
||||
log.info("Auto Form Detection model '{}' is installed and ready", activeModelId());
|
||||
} else {
|
||||
state = FormDetectionStatus.NOT_INSTALLED;
|
||||
}
|
||||
applyEndpointState();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gate the {@code form-detection} endpoint (which drives the tool tile): off with reason CONFIG
|
||||
* when the feature is disabled by the admin, off with reason DEPENDENCY when no model is ready,
|
||||
* otherwise on. Execution mode (browser/server) does not affect this - the tile is active
|
||||
* either way and the frontend chooses where to run.
|
||||
*/
|
||||
private void applyEndpointState() {
|
||||
if (!isFeatureEnabled()) {
|
||||
endpointConfiguration.disableEndpoint(ENDPOINT_KEY, DisableReason.CONFIG);
|
||||
} else if (state == FormDetectionStatus.READY && getActiveModelFile().isPresent()) {
|
||||
endpointConfiguration.enableEndpoint(ENDPOINT_KEY);
|
||||
} else {
|
||||
endpointConfiguration.disableEndpoint(ENDPOINT_KEY, DisableReason.DEPENDENCY);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isFeatureEnabled() {
|
||||
return applicationProperties.getFormDetection().isEnabled();
|
||||
}
|
||||
|
||||
/** Master on/off (admin). Persists and re-gates the endpoint immediately. */
|
||||
public synchronized void setEnabled(boolean enabled) {
|
||||
applicationProperties.getFormDetection().setEnabled(enabled);
|
||||
try {
|
||||
GeneralUtils.saveKeyToSettings("formDetection.enabled", enabled);
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not persist formDetection.enabled (state kept in memory)", e);
|
||||
}
|
||||
applyEndpointState();
|
||||
}
|
||||
|
||||
/** Set where detection runs: auto|browser|server (admin). Persists. */
|
||||
public synchronized void setExecutionMode(String mode) {
|
||||
String m = mode == null ? "auto" : mode.trim().toLowerCase(Locale.ROOT);
|
||||
if (!m.equals("auto") && !m.equals("browser") && !m.equals("server")) {
|
||||
throw new IllegalArgumentException("executionMode must be auto, browser or server");
|
||||
}
|
||||
applicationProperties.getFormDetection().setExecutionMode(m);
|
||||
try {
|
||||
GeneralUtils.saveKeyToSettings("formDetection.executionMode", m);
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not persist formDetection.executionMode (state kept in memory)", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and kick off a background download+verify+install. Returns immediately; callers poll
|
||||
* {@link #status()}.
|
||||
*
|
||||
* @throws IllegalArgumentException unknown/invalid model id or bad checksum format
|
||||
* @throws IllegalStateException no URL/checksum configured, or an install is already running
|
||||
*/
|
||||
public synchronized void startInstall(String modelId) {
|
||||
if (!SAFE_ID.matcher(modelId).matches()) {
|
||||
throw new IllegalArgumentException("Invalid model id: " + modelId);
|
||||
}
|
||||
ModelCatalogEntry entry =
|
||||
catalog.getById(modelId)
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException("Unknown model id: " + modelId));
|
||||
// URL + checksum come ONLY from the bundled catalog (trusted constants), never from the
|
||||
// request, so an admin cannot point the download at an arbitrary host (avoids SSRF).
|
||||
String url = entry.getOnnxUrl();
|
||||
String sha = entry.getSha256() == null ? null : entry.getSha256().toLowerCase(Locale.ROOT);
|
||||
if (StringUtils.isBlank(url) || StringUtils.isBlank(sha)) {
|
||||
throw new IllegalStateException(
|
||||
"Model '" + modelId + "' has no download URL/checksum configured yet");
|
||||
}
|
||||
String scheme = URI.create(url).getScheme();
|
||||
if (!"https".equalsIgnoreCase(scheme) && !"http".equalsIgnoreCase(scheme)) {
|
||||
throw new IllegalArgumentException("Model URL must be http(s): " + url);
|
||||
}
|
||||
if (!SHA256_HEX.matcher(sha).matches()) {
|
||||
throw new IllegalArgumentException("Checksum must be a 64-char hex SHA-256");
|
||||
}
|
||||
if (!installing.compareAndSet(false, true)) {
|
||||
throw new IllegalStateException("An install is already in progress");
|
||||
}
|
||||
state = FormDetectionStatus.DOWNLOADING;
|
||||
progress = 0;
|
||||
error = null;
|
||||
final String fUrl = url;
|
||||
final String fSha = sha;
|
||||
Thread.ofVirtual()
|
||||
.name("form-detection-install-" + modelId)
|
||||
.start(
|
||||
() -> {
|
||||
try {
|
||||
doInstall(modelId, entry, fUrl, fSha);
|
||||
} catch (Exception e) {
|
||||
log.error("Auto Form Detection install failed for {}", modelId, e);
|
||||
error = e.getMessage();
|
||||
// Keep a previously-installed model usable if the new one failed.
|
||||
state =
|
||||
getActiveModelFile().isPresent()
|
||||
? FormDetectionStatus.READY
|
||||
: FormDetectionStatus.FAILED;
|
||||
} finally {
|
||||
installing.set(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void doInstall(String modelId, ModelCatalogEntry entry, String url, String expectedSha)
|
||||
throws IOException {
|
||||
Path dir = modelDir();
|
||||
Files.createDirectories(dir);
|
||||
if (!isWritable(dir)) {
|
||||
throw new IOException("Model directory is not writable: " + dir);
|
||||
}
|
||||
Path base = dir.toRealPath();
|
||||
Path target = base.resolve(modelId + ".onnx").normalize();
|
||||
if (!target.startsWith(base)) {
|
||||
throw new IOException("Blocked path traversal for model id " + modelId);
|
||||
}
|
||||
|
||||
// Already downloaded and intact: skip the network fetch and just (re)activate it. Makes
|
||||
// switching between already-downloaded models instant instead of re-fetching tens of MB.
|
||||
if (Files.isRegularFile(target) && expectedSha.equals(sha256OfFile(target))) {
|
||||
log.info(
|
||||
"Model '{}' already present and verified; activating without re-download",
|
||||
modelId);
|
||||
activate(modelId, expectedSha);
|
||||
return;
|
||||
}
|
||||
|
||||
Path tmp = base.resolve(modelId + ".onnx.tmp");
|
||||
|
||||
MessageDigest digest;
|
||||
try {
|
||||
digest = MessageDigest.getInstance("SHA-256");
|
||||
} catch (Exception e) {
|
||||
throw new IOException("SHA-256 unavailable", e);
|
||||
}
|
||||
|
||||
HttpURLConnection conn = null;
|
||||
try {
|
||||
conn = (HttpURLConnection) URI.create(url).toURL().openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
conn.setRequestProperty("User-Agent", "Stirling-PDF-App");
|
||||
conn.setRequestProperty("Accept", "application/octet-stream");
|
||||
conn.setConnectTimeout(10000);
|
||||
conn.setReadTimeout(60000);
|
||||
int http = conn.getResponseCode();
|
||||
if (http != HttpURLConnection.HTTP_OK) {
|
||||
throw new IOException("Download failed: HTTP " + http + " from " + url);
|
||||
}
|
||||
long total =
|
||||
entry.getSizeBytes() > 0 ? entry.getSizeBytes() : conn.getContentLengthLong();
|
||||
try (InputStream in = conn.getInputStream();
|
||||
OutputStream out =
|
||||
Files.newOutputStream(tmp, CREATE, TRUNCATE_EXISTING, WRITE)) {
|
||||
byte[] buf = new byte[1 << 16];
|
||||
long read = 0;
|
||||
int n;
|
||||
while ((n = in.read(buf)) >= 0) {
|
||||
out.write(buf, 0, n);
|
||||
digest.update(buf, 0, n);
|
||||
read += n;
|
||||
if (total > 0) {
|
||||
progress = (int) Math.min(99, (read * 100) / total);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (conn != null) {
|
||||
conn.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
state = FormDetectionStatus.VERIFYING;
|
||||
byte[] actual = digest.digest();
|
||||
byte[] expected = HexFormat.of().parseHex(expectedSha);
|
||||
if (!MessageDigest.isEqual(actual, expected)) {
|
||||
Files.deleteIfExists(tmp);
|
||||
throw new IOException(
|
||||
"Checksum mismatch (expected "
|
||||
+ expectedSha
|
||||
+ " got "
|
||||
+ HexFormat.of().formatHex(actual)
|
||||
+ ")");
|
||||
}
|
||||
|
||||
try {
|
||||
Files.move(
|
||||
tmp,
|
||||
target,
|
||||
StandardCopyOption.ATOMIC_MOVE,
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (AtomicMoveNotSupportedException e) {
|
||||
Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
|
||||
activate(modelId, expectedSha);
|
||||
}
|
||||
|
||||
/** Mark a verified, on-disk model as the active one and (re)enable the feature. */
|
||||
private void activate(String modelId, String expectedSha) {
|
||||
applicationProperties.getFormDetection().setActiveModelId(modelId);
|
||||
try {
|
||||
GeneralUtils.saveKeyToSettings("formDetection.activeModelId", modelId);
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not persist formDetection.activeModelId (state kept in memory)", e);
|
||||
}
|
||||
activeSha = expectedSha;
|
||||
progress = 100;
|
||||
state = FormDetectionStatus.READY;
|
||||
applyEndpointState();
|
||||
log.info("Auto Form Detection model '{}' installed and ready", modelId);
|
||||
}
|
||||
|
||||
/** SHA-256 of an existing model file as lowercase hex, or {@code null} if it cannot be read. */
|
||||
private String sha256OfFile(Path file) {
|
||||
try (InputStream in = Files.newInputStream(file)) {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] buf = new byte[1 << 16];
|
||||
int n;
|
||||
while ((n = in.read(buf)) >= 0) {
|
||||
digest.update(buf, 0, n);
|
||||
}
|
||||
return HexFormat.of().formatHex(digest.digest());
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not hash existing model file {}", file, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove a model file; if it was the active one, disable the feature again. */
|
||||
public synchronized void deleteModel(String modelId) {
|
||||
if (installing.get()) {
|
||||
throw new IllegalStateException("Cannot uninstall while an install is in progress");
|
||||
}
|
||||
String id = StringUtils.isNotBlank(modelId) ? modelId : activeModelId();
|
||||
if (StringUtils.isBlank(id) || !SAFE_ID.matcher(id).matches()) {
|
||||
return;
|
||||
}
|
||||
Path base = modelDir().toAbsolutePath().normalize();
|
||||
Path file = base.resolve(id + ".onnx").normalize();
|
||||
if (!file.startsWith(base)) {
|
||||
return; // path-traversal guard (SAFE_ID already blocks it; this also satisfies CodeQL)
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(file);
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to delete model file {}", file, e);
|
||||
}
|
||||
if (id.equals(activeModelId())) {
|
||||
applicationProperties.getFormDetection().setActiveModelId("");
|
||||
try {
|
||||
GeneralUtils.saveKeyToSettings("formDetection.activeModelId", "");
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not clear formDetection.activeModelId", e);
|
||||
}
|
||||
activeSha = null;
|
||||
}
|
||||
if (getActiveModelFile().isEmpty()) {
|
||||
state = FormDetectionStatus.NOT_INSTALLED;
|
||||
error = null;
|
||||
}
|
||||
applyEndpointState();
|
||||
}
|
||||
|
||||
public ModelStatusResponse status() {
|
||||
Path dir = modelDir();
|
||||
List<String> installed = new ArrayList<>();
|
||||
if (Files.isDirectory(dir)) {
|
||||
try (DirectoryStream<Path> s = Files.newDirectoryStream(dir, "*.onnx")) {
|
||||
for (Path p : s) {
|
||||
String fn = p.getFileName().toString();
|
||||
installed.add(fn.substring(0, fn.length() - ".onnx".length()));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not list installed models in {}", dir, e);
|
||||
}
|
||||
}
|
||||
return new ModelStatusResponse(
|
||||
state.wire(),
|
||||
progress,
|
||||
activeModelId(),
|
||||
installed,
|
||||
error,
|
||||
isWritable(dir),
|
||||
catalog.getAll(),
|
||||
isFeatureEnabled(),
|
||||
applicationProperties.getFormDetection().getExecutionMode(),
|
||||
SERVER_ENGINE_AVAILABLE);
|
||||
}
|
||||
|
||||
public Optional<Path> getActiveModelFile() {
|
||||
String id = activeModelId();
|
||||
if (StringUtils.isBlank(id)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Path f = modelDir().resolve(id + ".onnx");
|
||||
return Files.isRegularFile(f) ? Optional.of(f) : Optional.empty();
|
||||
}
|
||||
|
||||
public Optional<ModelCatalogEntry> getActiveEntry() {
|
||||
return catalog.getById(activeModelId());
|
||||
}
|
||||
|
||||
public Optional<String> getActiveEtag() {
|
||||
return Optional.ofNullable(activeSha);
|
||||
}
|
||||
|
||||
public boolean isReady() {
|
||||
return isFeatureEnabled()
|
||||
&& state == FormDetectionStatus.READY
|
||||
&& getActiveModelFile().isPresent();
|
||||
}
|
||||
|
||||
private String activeModelId() {
|
||||
return applicationProperties.getFormDetection().getActiveModelId();
|
||||
}
|
||||
|
||||
private Path modelDir() {
|
||||
return Paths.get(runtimePathConfig.getFormDetectionModelPath());
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy any image-baked models (see {@code formDetection.preinstalledModelDir}) into the
|
||||
* writable model dir if not already present, and activate one when nothing is active yet. Lets
|
||||
* the Docker server image ship with FFDNet-S ready without an admin install. No-op when the dir
|
||||
* is unset or missing (desktop/local).
|
||||
*/
|
||||
private void seedPreinstalledModels() {
|
||||
String preDir = applicationProperties.getFormDetection().getPreinstalledModelDir();
|
||||
if (StringUtils.isBlank(preDir)) {
|
||||
return;
|
||||
}
|
||||
Path src = Paths.get(preDir);
|
||||
if (!Files.isDirectory(src)) {
|
||||
return;
|
||||
}
|
||||
Path dir = modelDir();
|
||||
try {
|
||||
Files.createDirectories(dir);
|
||||
} catch (IOException e) {
|
||||
log.warn("Cannot create model dir to seed pre-installed models: {}", e.getMessage());
|
||||
return;
|
||||
}
|
||||
if (!isWritable(dir)) {
|
||||
log.warn("Model dir {} not writable; skipping pre-installed model seeding", dir);
|
||||
return;
|
||||
}
|
||||
try (DirectoryStream<Path> models = Files.newDirectoryStream(src, "*.onnx")) {
|
||||
for (Path p : models) {
|
||||
String fn = p.getFileName().toString();
|
||||
String id = fn.substring(0, fn.length() - ".onnx".length());
|
||||
if (!SAFE_ID.matcher(id).matches() || catalog.getById(id).isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
Path target = dir.resolve(id + ".onnx");
|
||||
if (!Files.exists(target)) {
|
||||
Files.copy(p, target, StandardCopyOption.COPY_ATTRIBUTES);
|
||||
log.info("Seeded pre-installed Auto Form Detection model '{}'", id);
|
||||
}
|
||||
if (StringUtils.isBlank(activeModelId())) {
|
||||
applicationProperties.getFormDetection().setActiveModelId(id);
|
||||
try {
|
||||
GeneralUtils.saveKeyToSettings("formDetection.activeModelId", id);
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not persist seeded activeModelId: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to seed pre-installed models from {}: {}", src, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void sweepTempFiles() {
|
||||
Path dir = modelDir();
|
||||
if (!Files.isDirectory(dir)) {
|
||||
return;
|
||||
}
|
||||
try (DirectoryStream<Path> s = Files.newDirectoryStream(dir, "*.tmp")) {
|
||||
for (Path p : s) {
|
||||
try {
|
||||
Files.deleteIfExists(p);
|
||||
} catch (IOException ignored) {
|
||||
// best-effort sweep of interrupted downloads
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.debug("No stale form-detection temp files to sweep", e);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isWritable(Path dir) {
|
||||
try {
|
||||
Files.createDirectories(dir);
|
||||
if (!Files.isWritable(dir)) {
|
||||
return false;
|
||||
}
|
||||
Path probe = Files.createTempFile(dir, "fd-write-test", ".tmp");
|
||||
Files.deleteIfExists(probe);
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-40
@@ -43,10 +43,6 @@ public class IntegrationConfigService {
|
||||
private final OwnershipService ownership;
|
||||
private final SecretMasker secretMasker;
|
||||
private final ResourceGrantRepository grantRepository;
|
||||
// Bean-discovered extension points: features that understand a type contribute its config
|
||||
// schema and report what still references a config, without this module depending on them.
|
||||
private final List<IntegrationConfigValidator> validators;
|
||||
private final List<IntegrationConfigUsageCheck> usageChecks;
|
||||
|
||||
// ---- commands ----
|
||||
|
||||
@@ -70,21 +66,13 @@ public class IntegrationConfigService {
|
||||
? DefaultAccessPolicy.EXPLICIT_ONLY
|
||||
: request.defaultAccess());
|
||||
|
||||
// TEAM scope may omit the team id: default to the caller's own team so clients (the
|
||||
// portal) need not know it. assignOwnership still enforces admin-or-leader of that team.
|
||||
Long ownerTeamId = request.ownerTeamId();
|
||||
if (ownerTeamId == null && scope == OwnerScope.TEAM && currentUser.getTeam() != null) {
|
||||
ownerTeamId = currentUser.getTeam().getId();
|
||||
}
|
||||
ownership.assignOwnership(
|
||||
cfg,
|
||||
scope,
|
||||
ownerTeamId,
|
||||
request.ownerTeamId(),
|
||||
currentUser,
|
||||
() -> lockedServerExists(cfg.getIntegrationType()));
|
||||
Map<String, Object> config = secretMasker.sanitize(request.config());
|
||||
validateConfig(cfg.getIntegrationType(), config);
|
||||
cfg.setConfig(writeJson(config));
|
||||
cfg.setConfig(writeJson(secretMasker.sanitize(request.config())));
|
||||
return repository.save(cfg);
|
||||
}
|
||||
|
||||
@@ -113,10 +101,8 @@ public class IntegrationConfigService {
|
||||
cfg.setDefaultAccess(request.defaultAccess());
|
||||
}
|
||||
if (request.config() != null) {
|
||||
Map<String, Object> merged =
|
||||
secretMasker.merge(readJson(cfg.getConfig()), request.config());
|
||||
validateConfig(cfg.getIntegrationType(), merged);
|
||||
cfg.setConfig(writeJson(merged));
|
||||
cfg.setConfig(
|
||||
writeJson(secretMasker.merge(readJson(cfg.getConfig()), request.config())));
|
||||
}
|
||||
return repository.save(cfg);
|
||||
}
|
||||
@@ -127,15 +113,6 @@ public class IntegrationConfigService {
|
||||
if (!ownership.canManage(TYPE, cfg, currentUser)) {
|
||||
throw forbidden("You cannot manage this integration");
|
||||
}
|
||||
// Refuse to pull a connection out from under whatever still references it.
|
||||
List<String> usages =
|
||||
usageChecks.stream()
|
||||
.flatMap(check -> check.usagesOf(cfg.getId()).stream())
|
||||
.toList();
|
||||
if (!usages.isEmpty()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.CONFLICT, "Integration is in use by: " + String.join(", ", usages));
|
||||
}
|
||||
// Drop grants sharing this config so they do not dangle as dead rows.
|
||||
grantRepository.deleteByResourceTypeAndResourceId(TYPE, String.valueOf(cfg.getId()));
|
||||
repository.delete(cfg);
|
||||
@@ -211,19 +188,6 @@ public class IntegrationConfigService {
|
||||
|
||||
// ---- integration-specific glue ----
|
||||
|
||||
/** Runs every registered validator for the type; unknown types save free-form. */
|
||||
private void validateConfig(IntegrationType type, Map<String, Object> config) {
|
||||
for (IntegrationConfigValidator validator : validators) {
|
||||
if (validator.type() == type) {
|
||||
try {
|
||||
validator.validate(config == null ? Map.of() : config);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A non-admin can't create a personal config of a type an admin has locked at server scope. */
|
||||
private boolean lockedServerExists(IntegrationType type) {
|
||||
return repository.findByScope(OwnerScope.SERVER).stream()
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package stirling.software.proprietary.integration.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Reports what still references an integration config, so deletion can be refused instead of
|
||||
* pulling a connection out from under a live consumer. Implementations are beans discovered by
|
||||
* {@link IntegrationConfigService} (e.g. the policy subsystem reporting sources and pipelines that
|
||||
* reference a connection).
|
||||
*/
|
||||
public interface IntegrationConfigUsageCheck {
|
||||
|
||||
/** Human-readable labels of everything still using the config; empty when unreferenced. */
|
||||
List<String> usagesOf(long configId);
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package stirling.software.proprietary.integration.service;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
|
||||
/**
|
||||
* Validates one integration type's config map at save time. Implementations are beans discovered by
|
||||
* {@link IntegrationConfigService}, so the feature that understands a type (e.g. the policy S3
|
||||
* backend) owns its schema without the integration module depending on it. Types with no registered
|
||||
* validator save free-form.
|
||||
*/
|
||||
public interface IntegrationConfigValidator {
|
||||
|
||||
/** The type this validator understands. */
|
||||
IntegrationType type();
|
||||
|
||||
/**
|
||||
* Validates the config as it will be stored (secrets already sanitized/merged, so values are
|
||||
* real, never the redaction mask). Throws {@link IllegalArgumentException} on bad config.
|
||||
*/
|
||||
void validate(Map<String, Object> config);
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
package stirling.software.proprietary.model.api.ai.create;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class AiDocument {
|
||||
|
||||
private String title;
|
||||
private String subtitle;
|
||||
private String referenceNumber;
|
||||
private Style style;
|
||||
private List<Section> sections;
|
||||
|
||||
@Data
|
||||
public static class Style {
|
||||
private String primaryColor;
|
||||
private String backgroundColor;
|
||||
private String bodyTextColor;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Section {
|
||||
private String type;
|
||||
private String heading;
|
||||
private String body;
|
||||
private List<List<String>> pairs;
|
||||
private List<String> columns;
|
||||
private List<List<String>> rows;
|
||||
private List<String> totalRow;
|
||||
private List<String> items;
|
||||
private List<String> signatories;
|
||||
}
|
||||
}
|
||||
-20
@@ -120,7 +120,6 @@ public class PolicyController {
|
||||
throws IOException {
|
||||
stampPolicyAudit(definition);
|
||||
requireRunnable(definition);
|
||||
validateAdHocOutput(definition);
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
PolicyRunHandle handle =
|
||||
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP);
|
||||
@@ -141,7 +140,6 @@ public class PolicyController {
|
||||
throws IOException {
|
||||
stampPolicyAudit(definition);
|
||||
requireRunnable(definition);
|
||||
validateAdHocOutput(definition);
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
|
||||
SseEmitter emitter =
|
||||
@@ -532,24 +530,6 @@ public class PolicyController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Authorization-check an ad-hoc run's output while the caller's principal is present (this
|
||||
* request thread). The worker thread that later delivers carries no security context, so an S3
|
||||
* output's connection-access check would be skipped there; without this gate a caller could
|
||||
* reference another tenant's connection by id and write to it (confused deputy). Stored
|
||||
* policies are covered by save-time {@link PolicyValidator#validate} instead.
|
||||
*/
|
||||
private void validateAdHocOutput(PipelineDefinition definition) {
|
||||
if (definition.output() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
policyValidator.validateOutput(definition.output());
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ad-hoc runs (AI / one-off pipelines) are still editor activity, so their supplied documents
|
||||
* feed the same virtual editor source as stored editor policies, counted against the caller's
|
||||
|
||||
+1
-13
@@ -52,19 +52,7 @@ public class PolicyValidator {
|
||||
InputSpec spec = source.toInputSpec();
|
||||
inputSourceFor(spec).validate(spec);
|
||||
}
|
||||
validateOutput(policy.output());
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an output spec against its sink. Must be called on a request thread (caller's
|
||||
* principal present) so an S3 output's connection is authorization-checked against the caller -
|
||||
* ad-hoc runs are never persisted and so never hit {@link #validate(Policy)}, and the worker
|
||||
* thread that later delivers has no principal, so this is their only access gate.
|
||||
*
|
||||
* @throws IllegalArgumentException if the type is unknown or the config is invalid/inaccessible
|
||||
*/
|
||||
public void validateOutput(OutputSpec output) {
|
||||
outputSinkFor(output).validate(output);
|
||||
outputSinkFor(policy.output()).validate(policy.output());
|
||||
}
|
||||
|
||||
private PolicyTrigger triggerFor(TriggerConfig config) {
|
||||
|
||||
+13
-14
@@ -18,7 +18,6 @@ import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.s3.S3Config;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.s3.S3Identities;
|
||||
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
@@ -37,14 +36,15 @@ import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
* Reads input files from an Amazon S3 (or S3-compatible) bucket; each listed object is its own unit
|
||||
* of work, claimed through the {@link ResolveContext} ledger and tracked in place. Identity and
|
||||
* version gate come from {@link S3Identities}, so the steady-state sweep never downloads content.
|
||||
* Options: "connectionId" references the stored S3 connection (an {@code IntegrationConfig} owning
|
||||
* bucket, region, endpoint, and credentials - resolved by {@link S3ConnectionResolver}); "prefix"
|
||||
* (only keys starting with it are read) and "mode" are per-source, where mode is "consume"
|
||||
* (default: a processed object is deleted once every policy that claimed it has settled
|
||||
* successfully and it is still the version that ran; failures stay in place and are not retried
|
||||
* until they change) or "snapshot" (stateless, every run sees the full set). Keys ending in "/"
|
||||
* (folder placeholders) and keys with a dot-prefixed path segment are never picked up, mirroring
|
||||
* the folder source's hidden-file rule.
|
||||
* Options (see {@link S3Config}): "bucket" (required), "region" (default us-east-1), "prefix" (only
|
||||
* keys starting with it are read), "endpoint" (S3-compatible stores such as MinIO; path-style
|
||||
* addressing is used automatically), "accessKeyId" and "secretAccessKey" (required; requests are
|
||||
* never signed with the server's own AWS identity), and "mode" which is "consume" (default: a
|
||||
* processed object is deleted once every policy that claimed it has settled successfully and it is
|
||||
* still the version that ran; failures stay in place and are not retried until they change) or
|
||||
* "snapshot" (stateless, every run sees the full set). Keys ending in "/" (folder placeholders) and
|
||||
* keys with a dot-prefixed path segment are never picked up, mirroring the folder source's
|
||||
* hidden-file rule.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -55,7 +55,6 @@ public class S3InputSource implements InputSource {
|
||||
private static final String TYPE = "s3";
|
||||
|
||||
private final S3ConnectionPool connectionPool;
|
||||
private final S3ConnectionResolver connectionResolver;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
@@ -68,12 +67,12 @@ public class S3InputSource implements InputSource {
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails fast at save time: an unknown/disabled/unusable connection, bad config shape, a private
|
||||
* endpoint without the operator opt-in, or a bucket the connection cannot list.
|
||||
* Fails fast at save time: bad config shape, a private endpoint without the operator opt-in, or
|
||||
* a bucket the supplied credentials cannot list.
|
||||
*/
|
||||
@Override
|
||||
public void validate(InputSpec spec) {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Config config = S3Config.from(spec.options());
|
||||
try {
|
||||
connectionPool.clientFor(config).listObjectsV2(listRequest(config).maxKeys(1).build());
|
||||
} catch (SdkException e) {
|
||||
@@ -90,7 +89,7 @@ public class S3InputSource implements InputSource {
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Config config = S3Config.from(spec.options());
|
||||
S3Client client = connectionPool.clientFor(config);
|
||||
// A listing failure propagates so the sweep reads it as "could not list" (which vetoes
|
||||
// presence cleanup), never as "verifiably no objects".
|
||||
|
||||
+5
-7
@@ -27,7 +27,6 @@ import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3Config;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.s3.S3Identities;
|
||||
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
@@ -60,7 +59,6 @@ public class S3OutputSink implements PolicyOutputSink {
|
||||
private static final String TYPE = "s3";
|
||||
|
||||
private final S3ConnectionPool connectionPool;
|
||||
private final S3ConnectionResolver connectionResolver;
|
||||
private final ProcessedLedger processedLedger;
|
||||
|
||||
@Override
|
||||
@@ -74,19 +72,19 @@ public class S3OutputSink implements PolicyOutputSink {
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection resolution (including the saving user's right to use it) and endpoint guard only -
|
||||
* no network probe, since write-only credentials (s3:PutObject without s3:ListBucket) are a
|
||||
* legitimate setup for an output bucket and a listing probe would wrongly reject them.
|
||||
* Config shape and endpoint guard only - no network probe, since write-only credentials
|
||||
* (s3:PutObject without s3:ListBucket) are a legitimate setup for an output bucket and a
|
||||
* listing probe would wrongly reject them.
|
||||
*/
|
||||
@Override
|
||||
public void validate(OutputSpec spec) {
|
||||
connectionPool.clientFor(connectionResolver.resolve(spec.options()));
|
||||
connectionPool.clientFor(S3Config.from(spec.options()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResultFile> deliver(
|
||||
OutputDelivery delivery, List<Resource> outputs, OutputSpec spec) throws IOException {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Config config = S3Config.from(spec.options());
|
||||
S3Client client = connectionPool.clientFor(config);
|
||||
|
||||
List<ResultFile> results = new ArrayList<>();
|
||||
|
||||
-212
@@ -1,212 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.access.model.DefaultAccessPolicy;
|
||||
import stirling.software.proprietary.access.model.OwnerScope;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* One-time, idempotent extraction of legacy embedded S3 credentials into stored connections:
|
||||
* sources and policy outputs written before connections shipped carry bucket/credentials in their
|
||||
* own options; this rewrites each to reference a (deduplicated) S3 {@link IntegrationConfig} and
|
||||
* keeps only per-use options (prefix, mode). MUST be programmatic - the option JSON is encrypted at
|
||||
* the application layer, so no SQL migration can read it.
|
||||
*
|
||||
* <p>Idempotent by construction: rewritten rows no longer embed credentials, so re-runs find
|
||||
* nothing to do. Connections are deduplicated against both this run's extractions and existing S3
|
||||
* connections; a concurrent multi-node boot can at worst create a redundant connection row, never
|
||||
* corrupt a source. Ownership follows the owning row: team-scoped when the source/policy has a
|
||||
* team, server-scoped otherwise (single-operator self-hosted).
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class EmbeddedS3CredentialMigration {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
private static final List<String> CONNECTION_OPTIONS =
|
||||
List.of("bucket", "region", "endpoint", "accessKeyId", "secretAccessKey");
|
||||
// Field separator for the dedup key: a unit-separator control char that cannot appear in a
|
||||
// bucket/region/endpoint/credential, so distinct field sets can never collide.
|
||||
private static final char DELIMITER = '\u001f';
|
||||
|
||||
private final SourceStore sourceStore;
|
||||
private final PolicyStore policyStore;
|
||||
private final IntegrationConfigRepository connections;
|
||||
private final TeamRepository teamRepository;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
@Transactional
|
||||
public void migrate() {
|
||||
Map<String, IntegrationConfig> byCredentialKey = indexExistingConnections();
|
||||
int migrated = 0;
|
||||
for (Source source : sourceStore.all()) {
|
||||
if (!"s3".equals(source.type()) || !embedsCredentials(source.options())) {
|
||||
continue;
|
||||
}
|
||||
IntegrationConfig connection =
|
||||
connectionFor(source.options(), source.teamId(), byCredentialKey);
|
||||
sourceStore.save(withOptions(source, referencing(connection, source.options(), true)));
|
||||
migrated++;
|
||||
}
|
||||
for (Policy policy : policyStore.all()) {
|
||||
OutputSpec output = policy.output();
|
||||
if (!"s3".equals(output.type()) || !embedsCredentials(output.options())) {
|
||||
continue;
|
||||
}
|
||||
IntegrationConfig connection =
|
||||
connectionFor(output.options(), policy.teamId(), byCredentialKey);
|
||||
policyStore.save(
|
||||
withOutput(
|
||||
policy,
|
||||
new OutputSpec(
|
||||
output.type(),
|
||||
referencing(connection, output.options(), false))));
|
||||
migrated++;
|
||||
}
|
||||
if (migrated > 0) {
|
||||
log.info("Extracted embedded S3 credentials from {} row(s) into connections", migrated);
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean embedsCredentials(Map<String, Object> options) {
|
||||
return options.get("accessKeyId") != null;
|
||||
}
|
||||
|
||||
/** Reuses an existing connection with identical coordinates+credentials, else creates one. */
|
||||
private IntegrationConfig connectionFor(
|
||||
Map<String, Object> options, Long teamId, Map<String, IntegrationConfig> byKey) {
|
||||
String key = credentialKey(options);
|
||||
IntegrationConfig existing = byKey.get(key);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
IntegrationConfig connection = new IntegrationConfig();
|
||||
connection.setIntegrationType(IntegrationType.S3);
|
||||
connection.setName(connectionName(options, byKey));
|
||||
connection.setEnabled(true);
|
||||
connection.setLocked(false);
|
||||
connection.setDefaultAccess(DefaultAccessPolicy.EXPLICIT_ONLY);
|
||||
Team team = teamId == null ? null : teamRepository.findById(teamId).orElse(null);
|
||||
if (team != null) {
|
||||
connection.setScope(OwnerScope.TEAM);
|
||||
connection.setOwnerTeam(team);
|
||||
} else {
|
||||
// No team (teamless self-hosted, or a source whose team was since deleted): server
|
||||
// scope, i.e. admin-owned. An orphaned-team source's non-admin editor would then need
|
||||
// an admin to re-share the connection - acceptable for the narrow orphaned case.
|
||||
connection.setScope(OwnerScope.SERVER);
|
||||
}
|
||||
Map<String, Object> config = new LinkedHashMap<>();
|
||||
for (String option : CONNECTION_OPTIONS) {
|
||||
Object value = options.get(option);
|
||||
if (value != null && !value.toString().isBlank()) {
|
||||
config.put(option, value);
|
||||
}
|
||||
}
|
||||
connection.setConfig(OBJECT_MAPPER.writeValueAsString(config));
|
||||
IntegrationConfig saved = connections.save(connection);
|
||||
byKey.put(key, saved);
|
||||
return saved;
|
||||
}
|
||||
|
||||
/** The rewritten options: the connection reference plus per-use settings only. */
|
||||
private static Map<String, Object> referencing(
|
||||
IntegrationConfig connection, Map<String, Object> legacy, boolean keepMode) {
|
||||
Map<String, Object> options = new LinkedHashMap<>();
|
||||
options.put(S3ConnectionResolver.CONNECTION_ID_OPTION, connection.getId());
|
||||
Object prefix = legacy.get("prefix");
|
||||
if (prefix != null && !prefix.toString().isBlank()) {
|
||||
options.put("prefix", prefix);
|
||||
}
|
||||
Object mode = legacy.get("mode");
|
||||
if (keepMode && mode != null && !mode.toString().isBlank()) {
|
||||
options.put("mode", mode);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
private Map<String, IntegrationConfig> indexExistingConnections() {
|
||||
Map<String, IntegrationConfig> byKey = new LinkedHashMap<>();
|
||||
for (IntegrationConfig connection : connections.findAll()) {
|
||||
if (connection.getIntegrationType() != IntegrationType.S3) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
Map<String, Object> config =
|
||||
OBJECT_MAPPER.readValue(connection.getConfig(), Map.class);
|
||||
byKey.putIfAbsent(credentialKey(config), connection);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Skipping unreadable S3 connection {} while indexing: {}",
|
||||
connection.getId(),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
return byKey;
|
||||
}
|
||||
|
||||
private static String credentialKey(Map<String, Object> options) {
|
||||
StringBuilder key = new StringBuilder();
|
||||
for (String option : CONNECTION_OPTIONS) {
|
||||
Object value = options.get(option);
|
||||
key.append(value == null ? "" : value.toString().trim()).append(DELIMITER);
|
||||
}
|
||||
return key.toString();
|
||||
}
|
||||
|
||||
private static String connectionName(
|
||||
Map<String, Object> options, Map<String, IntegrationConfig> byKey) {
|
||||
String base = "S3: " + options.getOrDefault("bucket", "bucket");
|
||||
long sameName = byKey.values().stream().filter(c -> c.getName().startsWith(base)).count();
|
||||
return sameName == 0 ? base : base + " (" + (sameName + 1) + ")";
|
||||
}
|
||||
|
||||
private static Source withOptions(Source source, Map<String, Object> options) {
|
||||
return new Source(
|
||||
source.id(),
|
||||
source.name(),
|
||||
source.type(),
|
||||
options,
|
||||
source.enabled(),
|
||||
source.owner(),
|
||||
source.teamId());
|
||||
}
|
||||
|
||||
private static Policy withOutput(Policy policy, OutputSpec output) {
|
||||
return new Policy(
|
||||
policy.id(),
|
||||
policy.name(),
|
||||
policy.owner(),
|
||||
policy.enabled(),
|
||||
policy.trigger(),
|
||||
policy.sourceIds(),
|
||||
policy.steps(),
|
||||
output,
|
||||
policy.teamId());
|
||||
}
|
||||
}
|
||||
-56
@@ -1,56 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.proprietary.integration.service.IntegrationConfigUsageCheck;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
|
||||
/**
|
||||
* Reports the policy sources and pipeline outputs referencing an S3 connection, so the connection
|
||||
* cannot be deleted out from under them (mirrors {@code SourceController}'s referenced-source
|
||||
* delete guard). Scans in memory - fine at admin-dashboard scale, always consistent with the live
|
||||
* stores.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class PolicyS3ConnectionUsageCheck implements IntegrationConfigUsageCheck {
|
||||
|
||||
private final SourceStore sourceStore;
|
||||
private final PolicyStore policyStore;
|
||||
|
||||
@Override
|
||||
public List<String> usagesOf(long configId) {
|
||||
List<String> usages = new ArrayList<>();
|
||||
for (Source source : sourceStore.all()) {
|
||||
if (references(source.options(), configId)) {
|
||||
usages.add("source '" + source.name() + "'");
|
||||
}
|
||||
}
|
||||
for (Policy policy : policyStore.all()) {
|
||||
if (references(policy.output().options(), configId)) {
|
||||
usages.add("pipeline '" + policy.name() + "'");
|
||||
}
|
||||
}
|
||||
return usages;
|
||||
}
|
||||
|
||||
private static boolean references(Map<String, Object> options, long configId) {
|
||||
try {
|
||||
Long reference = S3ConnectionResolver.connectionId(options);
|
||||
return reference != null && reference == configId;
|
||||
} catch (IllegalArgumentException unparseable) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,12 +5,10 @@ import java.net.URISyntaxException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* The fully resolved connection settings the S3 input source and output sink run with - normally
|
||||
* produced by {@link S3ConnectionResolver} merging a stored connection (bucket, region, endpoint,
|
||||
* credentials) with per-use options (prefix, mode), or parsed directly from legacy options that
|
||||
* still embed credentials. Credentials are required: there is deliberately no fallback to the
|
||||
* server's own AWS credential chain, so user-supplied config can never borrow the host's identity.
|
||||
* {@code snapshot} is input-only and ignored by the sink.
|
||||
* Connection settings shared by the S3 input source and output sink, parsed from a spec's options
|
||||
* map. Credentials are required: there is deliberately no fallback to the server's own AWS
|
||||
* credential chain, so user-supplied config can never borrow the host's identity. {@code snapshot}
|
||||
* is input-only and ignored by the sink.
|
||||
*/
|
||||
public record S3Config(
|
||||
String bucket,
|
||||
|
||||
-151
@@ -1,151 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.security.core.userdetails.UserDetails;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Turns a source's or output's options into a full {@link S3Config} by dereferencing its {@code
|
||||
* connectionId} to a stored S3 {@link IntegrationConfig} (the connection owns bucket, region,
|
||||
* endpoint, and credentials; the options own per-use settings such as prefix and mode). Options
|
||||
* with no {@code connectionId} fall back to legacy embedded credentials, so rows written before
|
||||
* connections shipped keep working until {@link EmbeddedS3CredentialMigration} rewrites them.
|
||||
*
|
||||
* <p>When an authenticated caller is present (save-time validation), they must be allowed to use
|
||||
* the connection. Background sweeps and deliveries run with no caller and skip that check: the
|
||||
* referencing source or policy was access-checked when it was saved.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Transactional(readOnly = true)
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class S3ConnectionResolver {
|
||||
|
||||
static final String CONNECTION_ID_OPTION = "connectionId";
|
||||
private static final String PREFIX_OPTION = "prefix";
|
||||
private static final String MODE_OPTION = "mode";
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final IntegrationConfigRepository connections;
|
||||
private final OwnershipService ownership;
|
||||
private final UserService userService;
|
||||
|
||||
public S3Config resolve(Map<String, Object> options) {
|
||||
Long connectionId = connectionId(options);
|
||||
if (connectionId == null) {
|
||||
// Legacy embedded credentials, pending migration.
|
||||
return S3Config.from(options);
|
||||
}
|
||||
IntegrationConfig connection =
|
||||
connections
|
||||
.findById(connectionId)
|
||||
.filter(cfg -> cfg.getIntegrationType() == IntegrationType.S3)
|
||||
.filter(this::usableByCurrentUser)
|
||||
// Existence and access collapse into one error: a caller must not be able
|
||||
// to tell "no such connection" from "someone else's connection" and
|
||||
// enumerate ids. The id/name are never echoed.
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
"unknown or inaccessible s3 connection"));
|
||||
if (!connection.isEnabled()) {
|
||||
throw new IllegalArgumentException("s3 connection is disabled");
|
||||
}
|
||||
Map<String, Object> merged = new LinkedHashMap<>(connectionConfig(connection));
|
||||
copyPerUseOption(options, merged, PREFIX_OPTION);
|
||||
copyPerUseOption(options, merged, MODE_OPTION);
|
||||
return S3Config.from(merged);
|
||||
}
|
||||
|
||||
/** The {@code connectionId} option as a long, or null when the options are legacy-embedded. */
|
||||
static Long connectionId(Map<String, Object> options) {
|
||||
Object reference = options.get(CONNECTION_ID_OPTION);
|
||||
if (reference == null || (reference instanceof String s && s.isBlank())) {
|
||||
return null;
|
||||
}
|
||||
if (reference instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
try {
|
||||
return Long.valueOf(reference.toString().trim());
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException(
|
||||
"s3 'connectionId' is not a valid connection reference: " + reference);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the current caller may use this connection. With no principal - a background sweep or
|
||||
* delivery on a worker thread that carries no {@code SecurityContext} - access is treated as
|
||||
* already established: stored policies are validated with the caller present at save time, and
|
||||
* ad-hoc runs are validated on the request thread before dispatch (see {@code
|
||||
* PolicyValidator#validateOutput}). A missing principal must therefore never be the ONLY thing
|
||||
* standing between a caller and a connection, or the check becomes a confused deputy.
|
||||
*/
|
||||
private boolean usableByCurrentUser(IntegrationConfig connection) {
|
||||
User user = currentUser();
|
||||
return user == null || ownership.canUse(ResourceType.INTEGRATION_CONFIG, connection, user);
|
||||
}
|
||||
|
||||
// Mirrors ResourceAccessSecurity's principal resolution; null when unauthenticated.
|
||||
private User currentUser() {
|
||||
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
|
||||
if (auth == null || !auth.isAuthenticated()) {
|
||||
return null;
|
||||
}
|
||||
Object principal = auth.getPrincipal();
|
||||
if (principal instanceof User user) {
|
||||
return user;
|
||||
}
|
||||
if (principal instanceof UserDetails userDetails) {
|
||||
return userService.findByUsername(userDetails.getUsername()).orElse(null);
|
||||
}
|
||||
if (principal instanceof String username && !"anonymousUser".equals(username)) {
|
||||
return userService.findByUsername(username).orElse(null);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Map<String, Object> connectionConfig(IntegrationConfig connection) {
|
||||
String json = connection.getConfig();
|
||||
if (json == null || json.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return OBJECT_MAPPER.readValue(
|
||||
json, new TypeReference<LinkedHashMap<String, Object>>() {});
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(
|
||||
"s3 connection '" + connection.getName() + "' has unreadable config", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void copyPerUseOption(
|
||||
Map<String, Object> options, Map<String, Object> merged, String key) {
|
||||
Object value = options.get(key);
|
||||
if (value != null && !value.toString().isBlank()) {
|
||||
merged.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.cluster.s3.S3Clients;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.service.IntegrationConfigValidator;
|
||||
|
||||
/**
|
||||
* The S3 connection schema, enforced when an S3 {@link IntegrationType} config is saved: bucket and
|
||||
* credentials required, endpoint an http(s) URL that must not reach private addresses without the
|
||||
* operator opt-in - the same rules {@link S3ConnectionPool} enforces before signing, moved to save
|
||||
* time so a bad connection fails in the form rather than in a sweep.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class S3IntegrationValidator implements IntegrationConfigValidator {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@Override
|
||||
public IntegrationType type() {
|
||||
return IntegrationType.S3;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> config) {
|
||||
S3Config parsed = S3Config.from(config);
|
||||
if (parsed.endpoint() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
S3Clients.validateEndpointHost(
|
||||
URI.create(parsed.endpoint()),
|
||||
applicationProperties.getPolicies().isAllowPrivateS3Endpoints(),
|
||||
"S3 connection endpoint",
|
||||
"set policies.allowPrivateS3Endpoints=true to opt in (e.g. for a local"
|
||||
+ " MinIO).");
|
||||
} catch (IllegalStateException e) {
|
||||
throw new IllegalArgumentException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -37,7 +37,8 @@ import stirling.software.common.model.exception.UnsupportedProviderException;
|
||||
"stirling.software.proprietary.policy.ledger",
|
||||
"stirling.software.proprietary.accountlink",
|
||||
"stirling.software.proprietary.access.repository",
|
||||
"stirling.software.proprietary.integration.repository"
|
||||
"stirling.software.proprietary.integration.repository",
|
||||
"stirling.software.proprietary.classification.store"
|
||||
})
|
||||
@EntityScan({
|
||||
"stirling.software.proprietary.security.model",
|
||||
@@ -49,7 +50,8 @@ import stirling.software.common.model.exception.UnsupportedProviderException;
|
||||
"stirling.software.proprietary.policy.ledger",
|
||||
"stirling.software.proprietary.accountlink",
|
||||
"stirling.software.proprietary.access.model",
|
||||
"stirling.software.proprietary.integration.model"
|
||||
"stirling.software.proprietary.integration.model",
|
||||
"stirling.software.proprietary.classification.store"
|
||||
})
|
||||
public class DatabaseConfig {
|
||||
|
||||
|
||||
@@ -103,6 +103,7 @@ public class User implements UserDetails, Serializable {
|
||||
|
||||
@ElementCollection
|
||||
@MapKeyColumn(name = "setting_key")
|
||||
@Lob
|
||||
@Column(name = "setting_value", columnDefinition = "text")
|
||||
@CollectionTable(name = "user_settings", joinColumns = @JoinColumn(name = "user_id"))
|
||||
@JsonIgnore
|
||||
|
||||
-135
@@ -1,135 +0,0 @@
|
||||
package stirling.software.proprietary.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.hubspot.jinjava.Jinjava;
|
||||
import com.hubspot.jinjava.JinjavaConfig;
|
||||
|
||||
import stirling.software.proprietary.model.api.ai.create.AiDocument;
|
||||
|
||||
/** Renders an {@link AiDocument} to HTML using a Jinja template loaded from the classpath. */
|
||||
@Component
|
||||
public class AiDocumentHtmlRenderer {
|
||||
|
||||
private static final String TEMPLATE_PATH = "templates/ai/create/document.html.jinja2";
|
||||
|
||||
private static final Pattern SAFE_COLOR = Pattern.compile("^#[0-9a-fA-F]{6}$");
|
||||
|
||||
private final Jinjava jinjava;
|
||||
private final String template;
|
||||
|
||||
public AiDocumentHtmlRenderer() {
|
||||
JinjavaConfig config =
|
||||
JinjavaConfig.newBuilder().withNestedInterpretationEnabled(false).build();
|
||||
this.jinjava = new Jinjava(config);
|
||||
this.template = loadTemplate();
|
||||
}
|
||||
|
||||
public String render(AiDocument doc) {
|
||||
return jinjava.render(template, buildContext(doc));
|
||||
}
|
||||
|
||||
private static Map<String, Object> buildContext(AiDocument doc) {
|
||||
Map<String, Object> context = new LinkedHashMap<>();
|
||||
context.put("title", doc.getTitle());
|
||||
context.put("subtitle", doc.getSubtitle());
|
||||
context.put("reference_number", doc.getReferenceNumber());
|
||||
|
||||
AiDocument.Style style = doc.getStyle();
|
||||
if (style != null) {
|
||||
context.put("style_primary", safeColor(style.getPrimaryColor()));
|
||||
context.put("style_background", safeColor(style.getBackgroundColor()));
|
||||
context.put("style_body", safeColor(style.getBodyTextColor()));
|
||||
}
|
||||
|
||||
List<Map<String, Object>> sections = new ArrayList<>();
|
||||
if (doc.getSections() != null) {
|
||||
for (AiDocument.Section section : doc.getSections()) {
|
||||
if (section != null && section.getType() != null) {
|
||||
sections.add(buildSection(section));
|
||||
}
|
||||
}
|
||||
}
|
||||
context.put("sections", sections);
|
||||
return context;
|
||||
}
|
||||
|
||||
private static Map<String, Object> buildSection(AiDocument.Section section) {
|
||||
Map<String, Object> node = new LinkedHashMap<>();
|
||||
node.put("type", section.getType());
|
||||
node.put("heading", section.getHeading());
|
||||
switch (section.getType()) {
|
||||
case "text" -> node.put("paragraphs", paragraphs(section.getBody()));
|
||||
case "key_value" -> node.put("pairs", pairs(section.getPairs()));
|
||||
case "line_items" -> {
|
||||
node.put("columns", orEmpty(section.getColumns()));
|
||||
node.put("rows", orEmptyRows(section.getRows()));
|
||||
node.put("total_row", emptyToNull(section.getTotalRow()));
|
||||
}
|
||||
case "bullet_list" -> node.put("items", orEmpty(section.getItems()));
|
||||
case "signature" -> node.put("signatories", orEmpty(section.getSignatories()));
|
||||
default -> {}
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
private static List<String> paragraphs(String body) {
|
||||
String text = body == null ? "" : body;
|
||||
List<String> out = new ArrayList<>();
|
||||
for (String paragraph : text.split("\n\n")) {
|
||||
out.add(paragraph.replace("\n", " "));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static List<Map<String, String>> pairs(List<List<String>> pairs) {
|
||||
List<Map<String, String>> out = new ArrayList<>();
|
||||
if (pairs != null) {
|
||||
for (List<String> pair : pairs) {
|
||||
Map<String, String> node = new LinkedHashMap<>();
|
||||
node.put("label", pair.isEmpty() ? "" : pair.get(0));
|
||||
node.put("value", pair.size() < 2 ? "" : pair.get(1));
|
||||
out.add(node);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static List<String> orEmpty(List<String> values) {
|
||||
return values == null ? List.of() : values;
|
||||
}
|
||||
|
||||
private static List<List<String>> orEmptyRows(List<List<String>> rows) {
|
||||
return rows == null ? List.of() : rows;
|
||||
}
|
||||
|
||||
private static List<String> emptyToNull(List<String> values) {
|
||||
return values == null || values.isEmpty() ? null : values;
|
||||
}
|
||||
|
||||
private static String safeColor(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String trimmed = value.trim();
|
||||
return SAFE_COLOR.matcher(trimmed).matches() ? trimmed : null;
|
||||
}
|
||||
|
||||
private static String loadTemplate() {
|
||||
try {
|
||||
return new ClassPathResource(TEMPLATE_PATH).getContentAsString(StandardCharsets.UTF_8);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,46 @@
|
||||
[
|
||||
{
|
||||
"id": "ffdnet-s",
|
||||
"displayName": "CommonForms FFDNet-S (Small)",
|
||||
"description": "Small, fast detector with lower memory use - a good default for most forms. Finds text inputs, checkboxes, and signature fields.",
|
||||
"license": "CommonForms dataset CC-BY-4.0; Ultralytics-YOLO lineage AGPL-3.0. Downloaded on demand, never bundled.",
|
||||
"sizeBytes": 38370092,
|
||||
"onnxUrl": "https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/main/FFDNet-S.onnx",
|
||||
"sha256": "93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0",
|
||||
"inputSize": 1216,
|
||||
"resizeMode": "stretch",
|
||||
"padColor": [114, 114, 114],
|
||||
"channelOrder": "bgr",
|
||||
"normMean": [0.0, 0.0, 0.0],
|
||||
"normStd": [1.0, 1.0, 1.0],
|
||||
"outputLayout": "nc_first",
|
||||
"hasObjectness": false,
|
||||
"classNames": ["text", "choice", "signature"],
|
||||
"classFieldTypes": ["text", "checkbox", "signature"],
|
||||
"scoreThreshold": 0.3,
|
||||
"nms": "perClass",
|
||||
"iou": 0.45
|
||||
},
|
||||
{
|
||||
"id": "ffdnet-l",
|
||||
"displayName": "CommonForms FFDNet-L (Large)",
|
||||
"description": "Larger, higher-accuracy detector (~25M parameters) with better recall on dense or complex forms - uses more memory. Finds text inputs, checkboxes, and signature fields.",
|
||||
"license": "CommonForms dataset CC-BY-4.0; Ultralytics-YOLO lineage AGPL-3.0. Downloaded on demand, never bundled.",
|
||||
"sizeBytes": 101944542,
|
||||
"onnxUrl": "https://huggingface.co/jbarrow/FFDNet-L-cpu/resolve/main/FFDNet-L.onnx",
|
||||
"sha256": "e00c59edd9a5275dab5847d38f042c8ecc827063650c8aac22b0e486c414cd35",
|
||||
"inputSize": 1216,
|
||||
"resizeMode": "stretch",
|
||||
"padColor": [114, 114, 114],
|
||||
"channelOrder": "bgr",
|
||||
"normMean": [0.0, 0.0, 0.0],
|
||||
"normStd": [1.0, 1.0, 1.0],
|
||||
"outputLayout": "nc_first",
|
||||
"hasObjectness": false,
|
||||
"classNames": ["text", "choice", "signature"],
|
||||
"classFieldTypes": ["text", "checkbox", "signature"],
|
||||
"scoreThreshold": 0.3,
|
||||
"nms": "perClass",
|
||||
"iou": 0.45
|
||||
}
|
||||
]
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
package stirling.software.proprietary.classification;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
import stirling.software.proprietary.classification.store.ClassificationLabelStore;
|
||||
import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@DisplayName("ClassificationLabelsController")
|
||||
class ClassificationLabelsControllerTest {
|
||||
|
||||
private static final Long TEAM = 7L;
|
||||
|
||||
@Mock private PolicyManagementAuthority policyManagementAuthority;
|
||||
@Mock private UserServiceInterface userService;
|
||||
|
||||
private ClassificationLabelStore store;
|
||||
private ApplicationProperties applicationProperties;
|
||||
private ClassificationLabelsController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
store = new InProcessClassificationLabelStore();
|
||||
applicationProperties = new ApplicationProperties();
|
||||
controller =
|
||||
new ClassificationLabelsController(
|
||||
store, policyManagementAuthority, applicationProperties, userService);
|
||||
}
|
||||
|
||||
private static ClassificationLabels sample() {
|
||||
return new ClassificationLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
new ClassificationLabel("contract", "Contract", null)));
|
||||
}
|
||||
|
||||
private void loginEnabled(boolean enabled) {
|
||||
applicationProperties.getSecurity().setEnableLogin(enabled);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("GET returns 204 when the team has no labels")
|
||||
void getEmpty() {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
ResponseEntity<ClassificationLabels> response = controller.getTeamLabels();
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT then GET round-trips the team's labels (login disabled)")
|
||||
void saveThenGet() {
|
||||
loginEnabled(false);
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
|
||||
controller.saveTeamLabels(sample());
|
||||
ResponseEntity<ClassificationLabels> got = controller.getTeamLabels();
|
||||
|
||||
assertThat(got.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
assertThat(got.getBody()).isNotNull();
|
||||
assertThat(got.getBody().labels()).hasSize(2);
|
||||
assertThat(got.getBody().labels().getFirst().name()).isEqualTo("Invoice");
|
||||
assertThat(got.getBody().labels().getFirst().icon()).isEqualTo("receipt-long");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT is scoped per team")
|
||||
void perTeam() {
|
||||
loginEnabled(false);
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
controller.saveTeamLabels(sample());
|
||||
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(99L);
|
||||
assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT is rejected for a non-editor when login is enabled")
|
||||
void putForbiddenForNonEditor() {
|
||||
loginEnabled(true);
|
||||
when(policyManagementAuthority.canEditPolicies()).thenReturn(false);
|
||||
|
||||
assertThatThrownBy(() -> controller.saveTeamLabels(sample()))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.hasFieldOrPropertyWithValue("statusCode", HttpStatus.FORBIDDEN);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("PUT rejects an invalid label set with 400")
|
||||
void putInvalid() {
|
||||
loginEnabled(false);
|
||||
ClassificationLabels duplicate =
|
||||
new ClassificationLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", null),
|
||||
new ClassificationLabel("invoice", "Invoice", null)));
|
||||
|
||||
assertThatThrownBy(() -> controller.saveTeamLabels(duplicate))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.hasFieldOrPropertyWithValue("statusCode", HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("DELETE resets the team back to no stored labels")
|
||||
void deleteResets() {
|
||||
loginEnabled(false);
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
controller.saveTeamLabels(sample());
|
||||
|
||||
ResponseEntity<Void> response = controller.resetTeamLabels();
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
assertThat(controller.getTeamLabels().getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
|
||||
}
|
||||
}
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
package stirling.software.proprietary.classification.model;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@DisplayName("LabelsValidator")
|
||||
class LabelsValidatorTest {
|
||||
|
||||
private static ClassificationLabels labels(ClassificationLabel... labels) {
|
||||
return new ClassificationLabels(List.of(labels));
|
||||
}
|
||||
|
||||
private static ClassificationLabel label(String name) {
|
||||
return new ClassificationLabel(slug(name), name, null);
|
||||
}
|
||||
|
||||
private static String slug(String name) {
|
||||
return name.trim()
|
||||
.toLowerCase(Locale.ROOT)
|
||||
.replaceAll("[^a-z0-9]+", "-")
|
||||
.replaceAll("(^-|-$)", "");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("accepts a well-formed label set")
|
||||
void acceptsValid() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
label("Contract"));
|
||||
assertThatCode(() -> LabelsValidator.validate(set)).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("accepts an empty label set (reads as: use the default)")
|
||||
void acceptsEmpty() {
|
||||
assertThatCode(() -> LabelsValidator.validate(labels())).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects a null label set")
|
||||
void rejectsNull() {
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(null))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Labels are required");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects duplicate names (distinct ids)")
|
||||
void rejectsDuplicateNames() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice-a", "Invoice", null),
|
||||
new ClassificationLabel("invoice-b", "Invoice", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Duplicate label name");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects duplicate names differing only by case")
|
||||
void rejectsDuplicateNamesCaseInsensitive() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice-a", "Invoice", null),
|
||||
new ClassificationLabel("invoice-b", "INVOICE", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Duplicate label name");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects duplicate ids")
|
||||
void rejectsDuplicateIds() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel("invoice", "Invoice", null),
|
||||
new ClassificationLabel("invoice", "Sales invoice", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Duplicate label id");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects a blank name")
|
||||
void rejectsBlankName() {
|
||||
ClassificationLabels set = labels(new ClassificationLabel("blank", " ", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Label name must not be blank");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects a blank id")
|
||||
void rejectsBlankId() {
|
||||
ClassificationLabels set = labels(new ClassificationLabel(" ", "Invoice", null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Label id must not be blank");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects an over-long name")
|
||||
void rejectsOverLongName() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel(
|
||||
"x", "x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1), null));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("too long");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects an over-long icon (a null icon is fine)")
|
||||
void rejectsOverLongIcon() {
|
||||
ClassificationLabels set =
|
||||
labels(
|
||||
new ClassificationLabel(
|
||||
"invoice",
|
||||
"Invoice",
|
||||
"x".repeat(LabelsValidator.MAX_TEXT_LENGTH + 1)));
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(set))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("icon is too long");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects more labels than the cap")
|
||||
void rejectsTooManyLabels() {
|
||||
List<ClassificationLabel> tooMany =
|
||||
IntStream.rangeClosed(0, LabelsValidator.MAX_LABELS)
|
||||
.mapToObj(i -> label("label" + i))
|
||||
.toList();
|
||||
assertThatThrownBy(() -> LabelsValidator.validate(new ClassificationLabels(tooMany)))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("Too many labels");
|
||||
}
|
||||
}
|
||||
+34
-15
@@ -14,6 +14,7 @@ import static org.mockito.Mockito.when;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
@@ -26,8 +27,10 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.service.PdfMetadataService;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.proprietary.classification.ClassificationLabelProvider;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabel;
|
||||
import stirling.software.proprietary.classification.model.ClassificationLabels;
|
||||
import stirling.software.proprietary.classification.store.InProcessClassificationLabelStore;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.service.AiEngineClient;
|
||||
import stirling.software.proprietary.service.PdfContentExtractor;
|
||||
|
||||
@@ -39,16 +42,22 @@ import tools.jackson.databind.json.JsonMapper;
|
||||
@MockitoSettings(strictness = Strictness.LENIENT)
|
||||
class ClassifyLabelControllerTest {
|
||||
|
||||
private static final Long TEAM = 7L;
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private PdfContentExtractor pdfContentExtractor;
|
||||
@Mock private PdfMetadataService pdfMetadataService;
|
||||
@Mock private AiEngineClient aiEngineClient;
|
||||
@Mock private PolicyManagementAuthority policyManagementAuthority;
|
||||
|
||||
private final ObjectMapper objectMapper = JsonMapper.builder().build();
|
||||
private InProcessClassificationLabelStore labelStore;
|
||||
private ClassifyLabelController controller;
|
||||
|
||||
private void withLabels(List<ClassificationLabel> labels) {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
labelStore = new InProcessClassificationLabelStore();
|
||||
controller =
|
||||
new ClassifyLabelController(
|
||||
pdfDocumentFactory,
|
||||
@@ -57,8 +66,9 @@ class ClassifyLabelControllerTest {
|
||||
pdfMetadataService,
|
||||
aiEngineClient,
|
||||
objectMapper,
|
||||
ClassificationLabelProvider.withLabels(labels),
|
||||
null);
|
||||
null,
|
||||
labelStore,
|
||||
policyManagementAuthority);
|
||||
}
|
||||
|
||||
private void stubSinglePageDocument() throws Exception {
|
||||
@@ -88,7 +98,12 @@ class ClassifyLabelControllerTest {
|
||||
|
||||
@Test
|
||||
void classifyAndLabel_writesClassificationWithoutOutcome() throws Exception {
|
||||
withLabels(List.of(new ClassificationLabel("invoice", "Invoice", null)));
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
labelStore.save(
|
||||
TEAM,
|
||||
new ClassificationLabels(
|
||||
List.of(new ClassificationLabel("invoice", "Invoice", null))),
|
||||
"admin");
|
||||
|
||||
stubSinglePageDocument();
|
||||
|
||||
@@ -103,12 +118,16 @@ class ClassifyLabelControllerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyAndLabel_sendsLabelIdsAndNames() throws Exception {
|
||||
withLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
new ClassificationLabel("contract", "Contract", null),
|
||||
new ClassificationLabel("timesheet", "Timesheet", null)));
|
||||
void classifyAndLabel_sendsTeamLabelIdsAndNames() throws Exception {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
labelStore.save(
|
||||
TEAM,
|
||||
new ClassificationLabels(
|
||||
List.of(
|
||||
new ClassificationLabel("invoice", "Invoice", "receipt-long"),
|
||||
new ClassificationLabel("contract", "Contract", null),
|
||||
new ClassificationLabel("timesheet", "Timesheet", null))),
|
||||
"admin");
|
||||
|
||||
stubSinglePageDocument();
|
||||
|
||||
@@ -133,13 +152,13 @@ class ClassifyLabelControllerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void classifyAndLabel_skipsClassificationWhenNoLabels() throws Exception {
|
||||
withLabels(List.of());
|
||||
void classifyAndLabel_skipsClassificationWhenNothingStored() throws Exception {
|
||||
when(policyManagementAuthority.currentUserTeamId()).thenReturn(TEAM);
|
||||
|
||||
stubSinglePageDocument();
|
||||
|
||||
// No vocabulary, and the engine holds no default of its own, so the file is passed through
|
||||
// unlabelled: neither the engine nor the metadata write is invoked.
|
||||
// No team labels stored, and the engine holds no default of its own, so the file is passed
|
||||
// through unlabelled: neither the engine nor the metadata write is invoked.
|
||||
verify(aiEngineClient, never()).post(anyString(), anyString(), any());
|
||||
verify(pdfMetadataService, never())
|
||||
.setClassificationMetadata(any(PDDocument.class), anyString());
|
||||
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package stirling.software.proprietary.formdetection.catalog;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class ModelCatalogServiceTest {
|
||||
|
||||
@Test
|
||||
void loadsBundledCatalogWithSpecDefaults() {
|
||||
ModelCatalogService service = new ModelCatalogService(JsonMapper.builder().build());
|
||||
service.load();
|
||||
|
||||
List<ModelCatalogEntry> all = service.getAll();
|
||||
assertTrue(all.size() >= 2, "catalog should ship with at least two entries");
|
||||
assertTrue(service.getById("ffdnet-s").isPresent());
|
||||
|
||||
ModelCatalogEntry l = service.getById("ffdnet-l").orElseThrow();
|
||||
assertEquals(3, l.getClassNames().size());
|
||||
assertEquals(3, l.getClassFieldTypes().size());
|
||||
assertTrue(l.getInputSize() > 0);
|
||||
|
||||
// Model-free distribution: the jar bundles no weights. Every entry instead carries a
|
||||
// download URL and a SHA-256 so the model is fetched and integrity-verified on demand.
|
||||
for (ModelCatalogEntry e : all) {
|
||||
assertNotNull(e.getOnnxUrl(), e.getId() + " must declare a download URL");
|
||||
assertFalse(e.getOnnxUrl().isBlank(), e.getId() + " must declare a download URL");
|
||||
assertNotNull(e.getSha256(), e.getId() + " must declare a SHA-256 checksum");
|
||||
assertFalse(e.getSha256().isBlank(), e.getId() + " must declare a SHA-256 checksum");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownIdReturnsEmpty() {
|
||||
ModelCatalogService service = new ModelCatalogService(JsonMapper.builder().build());
|
||||
service.load();
|
||||
assertTrue(service.getById("does-not-exist").isEmpty());
|
||||
assertTrue(service.getById(null).isEmpty());
|
||||
}
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
package stirling.software.proprietary.formdetection.controller;
|
||||
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.proprietary.formdetection.inference.OnnxFormDetector;
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
import stirling.software.proprietary.formdetection.render.PageRasterizer;
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
class FormDetectionControllerTest {
|
||||
|
||||
private MockMvc mvc(
|
||||
FormDetectionModelManager manager,
|
||||
OnnxFormDetector detector,
|
||||
PageRasterizer rasterizer) {
|
||||
FormDetectionController controller =
|
||||
new FormDetectionController(
|
||||
manager,
|
||||
detector,
|
||||
rasterizer,
|
||||
Mockito.mock(CustomPDFDocumentFactory.class),
|
||||
Mockito.mock(TempFileManager.class));
|
||||
return MockMvcBuilders.standaloneSetup(controller).build();
|
||||
}
|
||||
|
||||
private MockMultipartFile pdf() {
|
||||
return new MockMultipartFile("file", "test.pdf", "application/pdf", "%PDF-1.4".getBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectReturns503WhenModelNotReady() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.isReady()).thenReturn(false);
|
||||
|
||||
mvc(manager, Mockito.mock(OnnxFormDetector.class), Mockito.mock(PageRasterizer.class))
|
||||
.perform(multipart("/api/v1/ai/form-detection/detect").file(pdf()))
|
||||
.andExpect(status().isServiceUnavailable())
|
||||
.andExpect(jsonPath("$.reason").value("DEPENDENCY"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectReturnsEmptyDetectionsForBlankRender() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.isReady()).thenReturn(true);
|
||||
Mockito.when(manager.getActiveEntry()).thenReturn(Optional.of(new ModelCatalogEntry()));
|
||||
|
||||
PageRasterizer rasterizer = Mockito.mock(PageRasterizer.class);
|
||||
Mockito.when(rasterizer.rasterize(Mockito.any(), Mockito.anyInt()))
|
||||
.thenReturn(List.of()); // no pages -> no detections, detector never called
|
||||
|
||||
mvc(manager, Mockito.mock(OnnxFormDetector.class), rasterizer)
|
||||
.perform(multipart("/api/v1/ai/form-detection/detect").file(pdf()))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.detections").isArray())
|
||||
.andExpect(jsonPath("$.detections").isEmpty());
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package stirling.software.proprietary.formdetection.controller;
|
||||
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelStatusResponse;
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
class FormDetectionModelControllerTest {
|
||||
|
||||
private ModelStatusResponse notInstalled() {
|
||||
return new ModelStatusResponse(
|
||||
"not_installed", 0, "", List.of(), null, true, List.of(), true, "auto", true);
|
||||
}
|
||||
|
||||
private MockMvc mvc(FormDetectionModelManager manager) {
|
||||
return MockMvcBuilders.standaloneSetup(new FormDetectionModelController(manager)).build();
|
||||
}
|
||||
|
||||
@Test
|
||||
void statusReturnsJson() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.status()).thenReturn(notInstalled());
|
||||
|
||||
mvc(manager)
|
||||
.perform(get("/api/v1/ai/form-detection-model/status"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.status").value("not_installed"))
|
||||
.andExpect(jsonPath("$.writable").value(true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void installWithBlankModelIdReturns400() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.status()).thenReturn(notInstalled());
|
||||
|
||||
mvc(manager)
|
||||
.perform(
|
||||
post("/api/v1/ai/form-detection-model/install")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"modelId\":\"\"}"))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
void installValidReturns202() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.status()).thenReturn(notInstalled());
|
||||
|
||||
mvc(manager)
|
||||
.perform(
|
||||
post("/api/v1/ai/form-detection-model/install")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"modelId\":\"ffdnet-s\"}"))
|
||||
.andExpect(status().isAccepted());
|
||||
}
|
||||
|
||||
@Test
|
||||
void installWhileBusyReturns409() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.status()).thenReturn(notInstalled());
|
||||
Mockito.doThrow(new IllegalStateException("An install is already in progress"))
|
||||
.when(manager)
|
||||
.startInstall(Mockito.eq("ffdnet-s"));
|
||||
|
||||
mvc(manager)
|
||||
.perform(
|
||||
post("/api/v1/ai/form-detection-model/install")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"modelId\":\"ffdnet-s\"}"))
|
||||
.andExpect(status().isConflict());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteReturnsStatus() throws Exception {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.status()).thenReturn(notInstalled());
|
||||
|
||||
mvc(manager)
|
||||
.perform(delete("/api/v1/ai/form-detection-model"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.status").value("not_installed"));
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package stirling.software.proprietary.formdetection.controller;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.support.ResourceRegion;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpRange;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
|
||||
import stirling.software.proprietary.formdetection.service.FormDetectionModelManager;
|
||||
|
||||
/**
|
||||
* Tests the serve controller's logic directly (status, headers, region bounds). Serializing a
|
||||
* ResourceRegion needs the full Spring resource converters, which standaloneSetup does not wire, so
|
||||
* exercising the method directly is both simpler and converter-independent.
|
||||
*/
|
||||
class FormDetectionModelServeControllerTest {
|
||||
|
||||
private FormDetectionModelManager managerWith(Path model) {
|
||||
FormDetectionModelManager manager = Mockito.mock(FormDetectionModelManager.class);
|
||||
Mockito.when(manager.getActiveModelFile())
|
||||
.thenReturn(model == null ? Optional.empty() : Optional.of(model));
|
||||
Mockito.when(manager.getActiveEtag()).thenReturn(Optional.of("a".repeat(64)));
|
||||
return manager;
|
||||
}
|
||||
|
||||
@Test
|
||||
void servesFullModelWithPublicCacheAndAcceptRanges(@TempDir Path dir) throws Exception {
|
||||
Path model = dir.resolve("m.onnx");
|
||||
Files.write(model, "abcdefghij".getBytes());
|
||||
|
||||
ResponseEntity<Object> resp =
|
||||
new FormDetectionModelServeController(managerWith(model))
|
||||
.serveModel(new HttpHeaders());
|
||||
|
||||
assertEquals(HttpStatus.OK, resp.getStatusCode());
|
||||
assertEquals("bytes", resp.getHeaders().getFirst(HttpHeaders.ACCEPT_RANGES));
|
||||
assertNotNull(resp.getHeaders().getFirst(HttpHeaders.CACHE_CONTROL));
|
||||
assertTrue(resp.getHeaders().getFirst(HttpHeaders.CACHE_CONTROL).contains("public"));
|
||||
assertNotNull(resp.getHeaders().getETag());
|
||||
assertTrue(resp.getBody() instanceof Resource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void servesRangeRequestAsSingleRegion(@TempDir Path dir) throws Exception {
|
||||
Path model = dir.resolve("m.onnx");
|
||||
Files.write(model, "abcdefghij".getBytes());
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setRange(List.of(HttpRange.createByteRange(0, 3)));
|
||||
|
||||
ResponseEntity<Object> resp =
|
||||
new FormDetectionModelServeController(managerWith(model)).serveModel(headers);
|
||||
|
||||
assertEquals(HttpStatus.PARTIAL_CONTENT, resp.getStatusCode());
|
||||
assertEquals("bytes", resp.getHeaders().getFirst(HttpHeaders.ACCEPT_RANGES));
|
||||
assertTrue(resp.getBody() instanceof ResourceRegion, "body should be a single region");
|
||||
ResourceRegion region = (ResourceRegion) resp.getBody();
|
||||
assertEquals(0L, region.getPosition());
|
||||
assertEquals(4L, region.getCount()); // bytes 0-3 inclusive
|
||||
}
|
||||
|
||||
@Test
|
||||
void returns404WhenNoModelInstalled() throws Exception {
|
||||
ResponseEntity<Object> resp =
|
||||
new FormDetectionModelServeController(managerWith(null))
|
||||
.serveModel(new HttpHeaders());
|
||||
assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode());
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package stirling.software.proprietary.formdetection.inference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
|
||||
class YoloTest {
|
||||
|
||||
private ModelCatalogEntry spec() {
|
||||
ModelCatalogEntry e = new ModelCatalogEntry();
|
||||
e.setInputSize(10);
|
||||
e.setResizeMode("letterbox");
|
||||
e.setChannelOrder("rgb");
|
||||
e.setOutputLayout("nc_first");
|
||||
e.setHasObjectness(false);
|
||||
e.setClassNames(List.of("text", "choice"));
|
||||
e.setClassFieldTypes(List.of("text", "checkbox"));
|
||||
e.setNms("perClass");
|
||||
e.setIou(0.5f);
|
||||
return e;
|
||||
}
|
||||
|
||||
@Test
|
||||
void decodeThresholdsAndSuppressesOverlaps() {
|
||||
ModelCatalogEntry spec = spec();
|
||||
// identity transform (scale 1, no pad), 10x10 source
|
||||
Yolo.Preprocessed pre = new Yolo.Preprocessed(new float[0], 10, 1f, 1f, 0, 0, 10, 10);
|
||||
|
||||
// nc_first layout [channels=6][anchors=3], data[c*anchors + a]
|
||||
// box A (cx5,cy5,w4,h4) twice (overlapping) + box B (cx8,cy8,w2,h2)
|
||||
float[] data = {
|
||||
5,
|
||||
5,
|
||||
8, // cx
|
||||
5,
|
||||
5,
|
||||
8, // cy
|
||||
4,
|
||||
4,
|
||||
2, // w
|
||||
4,
|
||||
4,
|
||||
2, // h
|
||||
0.9f,
|
||||
0.8f,
|
||||
0.7f, // text score
|
||||
0.1f,
|
||||
0.1f,
|
||||
0.1f // choice score
|
||||
};
|
||||
Yolo.RawOutput out = new Yolo.RawOutput(data, 6, 3);
|
||||
|
||||
List<Yolo.Detection> dets = Yolo.decode(out, spec, pre, 0.5f);
|
||||
|
||||
// a0 (box A, 0.9) kept; a1 (box A', 0.8) suppressed by NMS; a2 (box B, 0.7) kept
|
||||
assertEquals(2, dets.size());
|
||||
|
||||
Yolo.Detection a = dets.get(0);
|
||||
assertEquals(0, a.classId());
|
||||
assertEquals(0.9f, a.score(), 1e-5);
|
||||
assertEquals(3f, a.x(), 1e-4);
|
||||
assertEquals(3f, a.y(), 1e-4);
|
||||
assertEquals(4f, a.w(), 1e-4);
|
||||
assertEquals(4f, a.h(), 1e-4);
|
||||
|
||||
Yolo.Detection b = dets.get(1);
|
||||
assertEquals(0.7f, b.score(), 1e-5);
|
||||
assertEquals(7f, b.x(), 1e-4);
|
||||
assertEquals(7f, b.y(), 1e-4);
|
||||
assertEquals(2f, b.w(), 1e-4);
|
||||
assertEquals(2f, b.h(), 1e-4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void decodeIsDeterministic() {
|
||||
ModelCatalogEntry spec = spec();
|
||||
Yolo.Preprocessed pre = new Yolo.Preprocessed(new float[0], 10, 1f, 1f, 0, 0, 10, 10);
|
||||
float[] data = {5, 5, 8, 5, 5, 8, 4, 4, 2, 4, 4, 2, 0.9f, 0.8f, 0.7f, 0.1f, 0.1f, 0.1f};
|
||||
Yolo.RawOutput out = new Yolo.RawOutput(data, 6, 3);
|
||||
assertEquals(
|
||||
Yolo.decode(out, spec, pre, 0.5f).toString(),
|
||||
Yolo.decode(out, spec, pre, 0.5f).toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void thresholdDropsLowScores() {
|
||||
ModelCatalogEntry spec = spec();
|
||||
Yolo.Preprocessed pre = new Yolo.Preprocessed(new float[0], 10, 1f, 1f, 0, 0, 10, 10);
|
||||
float[] data = {5, 5, 8, 5, 5, 8, 4, 4, 2, 4, 4, 2, 0.9f, 0.8f, 0.7f, 0.1f, 0.1f, 0.1f};
|
||||
Yolo.RawOutput out = new Yolo.RawOutput(data, 6, 3);
|
||||
// threshold above every score -> nothing survives
|
||||
assertEquals(0, Yolo.decode(out, spec, pre, 0.95f).size());
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package stirling.software.proprietary.formdetection.render;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.formdetection.inference.Yolo;
|
||||
import stirling.software.proprietary.formdetection.model.DetectedField;
|
||||
|
||||
class CoordinateMapperTest {
|
||||
|
||||
@Test
|
||||
void mapsBitmapPixelsToPdfPointsWithYFlip() {
|
||||
// 200x300pt page rendered at 2 px/pt (400x600 px)
|
||||
PageRasterizer.RasterPage page =
|
||||
new PageRasterizer.RasterPage(0, new byte[0], 400, 600, 200f, 300f, 2f, 2f);
|
||||
// detection at top-left (10,20) px, 40x60 px
|
||||
Yolo.Detection d = new Yolo.Detection(0, 0.9f, 10f, 20f, 40f, 60f);
|
||||
|
||||
DetectedField.RectPt r = CoordinateMapper.toPdfPoints(d, page);
|
||||
|
||||
assertEquals(5.0, r.x(), 1e-4); // 10/2
|
||||
assertEquals(20.0, r.w(), 1e-4); // 40/2
|
||||
assertEquals(30.0, r.h(), 1e-4); // 60/2
|
||||
// Y flip: pageHeight - (yTopPx/scale) - hPt = 300 - 10 - 30
|
||||
assertEquals(260.0, r.y(), 1e-4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void clampsToPageBounds() {
|
||||
PageRasterizer.RasterPage page =
|
||||
new PageRasterizer.RasterPage(0, new byte[0], 200, 200, 100f, 100f, 2f, 2f);
|
||||
// box partly off the right/bottom edge in px
|
||||
Yolo.Detection d = new Yolo.Detection(0, 0.5f, 180f, 0f, 60f, 40f);
|
||||
|
||||
DetectedField.RectPt r = CoordinateMapper.toPdfPoints(d, page);
|
||||
|
||||
// x = 90pt, width clamped to 100-90 = 10pt
|
||||
assertEquals(90.0, r.x(), 1e-4);
|
||||
assertEquals(10.0, r.w(), 1e-4);
|
||||
// stays within the page
|
||||
org.junit.jupiter.api.Assertions.assertEquals(true, r.x() + r.w() <= 100.0 + 1e-6);
|
||||
org.junit.jupiter.api.Assertions.assertEquals(true, r.y() >= -1e-6);
|
||||
}
|
||||
}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
package stirling.software.proprietary.formdetection.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.formdetection.catalog.ModelCatalogService;
|
||||
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
|
||||
|
||||
class FormDetectionModelManagerTest {
|
||||
|
||||
private HttpServer server;
|
||||
private byte[] modelBytes;
|
||||
private String modelSha;
|
||||
private int port;
|
||||
|
||||
@BeforeEach
|
||||
void startServer() throws Exception {
|
||||
modelBytes = "fake-onnx-model-content-1234567890".getBytes();
|
||||
modelSha =
|
||||
HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(modelBytes));
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
port = server.getAddress().getPort();
|
||||
server.createContext(
|
||||
"/model.onnx",
|
||||
ex -> {
|
||||
ex.sendResponseHeaders(200, modelBytes.length);
|
||||
ex.getResponseBody().write(modelBytes);
|
||||
ex.close();
|
||||
});
|
||||
server.start();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void stopServer() {
|
||||
if (server != null) {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
private ModelCatalogEntry entry(String url, String sha) {
|
||||
ModelCatalogEntry e = new ModelCatalogEntry();
|
||||
e.setId("test-model");
|
||||
e.setOnnxUrl(url);
|
||||
e.setSha256(sha);
|
||||
e.setSizeBytes(modelBytes.length);
|
||||
return e;
|
||||
}
|
||||
|
||||
private FormDetectionModelManager manager(
|
||||
Path dir, ModelCatalogEntry entry, EndpointConfiguration ep) {
|
||||
RuntimePathConfig paths = Mockito.mock(RuntimePathConfig.class);
|
||||
Mockito.when(paths.getFormDetectionModelPath()).thenReturn(dir.toString());
|
||||
ModelCatalogService catalog = Mockito.mock(ModelCatalogService.class);
|
||||
Mockito.when(catalog.getById("test-model")).thenReturn(Optional.of(entry));
|
||||
Mockito.when(catalog.getById(Mockito.argThat(s -> !"test-model".equals(s))))
|
||||
.thenReturn(Optional.empty());
|
||||
Mockito.when(catalog.getAll()).thenReturn(List.of(entry));
|
||||
return new FormDetectionModelManager(paths, catalog, new ApplicationProperties(), ep);
|
||||
}
|
||||
|
||||
private void awaitState(FormDetectionModelManager m, String wire, long timeoutMs)
|
||||
throws InterruptedException {
|
||||
long deadline = System.currentTimeMillis() + timeoutMs;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
if (wire.equals(m.status().getStatus())) {
|
||||
return;
|
||||
}
|
||||
Thread.sleep(25);
|
||||
}
|
||||
fail("Timed out waiting for state '" + wire + "', was '" + m.status().getStatus() + "'");
|
||||
}
|
||||
|
||||
@Test
|
||||
void installsDownloadsVerifiesAndPublishesAtomically(@TempDir Path dir) throws Exception {
|
||||
EndpointConfiguration ep = Mockito.mock(EndpointConfiguration.class);
|
||||
FormDetectionModelManager m =
|
||||
manager(dir, entry("http://127.0.0.1:" + port + "/model.onnx", modelSha), ep);
|
||||
|
||||
m.startInstall("test-model");
|
||||
awaitState(m, "ready", 5000);
|
||||
|
||||
Path onnx = dir.resolve("test-model.onnx");
|
||||
assertTrue(Files.exists(onnx), "model file should be published");
|
||||
assertArrayEquals(modelBytes, Files.readAllBytes(onnx));
|
||||
assertFalse(Files.exists(dir.resolve("test-model.onnx.tmp")), "temp file should be gone");
|
||||
assertTrue(m.isReady());
|
||||
Mockito.verify(ep).enableEndpoint("form-detection");
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsChecksumMismatchAndLeavesNoFile(@TempDir Path dir) throws Exception {
|
||||
EndpointConfiguration ep = Mockito.mock(EndpointConfiguration.class);
|
||||
FormDetectionModelManager m =
|
||||
manager(dir, entry("http://127.0.0.1:" + port + "/model.onnx", "0".repeat(64)), ep);
|
||||
|
||||
m.startInstall("test-model");
|
||||
awaitState(m, "failed", 5000);
|
||||
|
||||
assertFalse(Files.exists(dir.resolve("test-model.onnx")), "no model on mismatch");
|
||||
assertFalse(Files.exists(dir.resolve("test-model.onnx.tmp")), "temp cleaned up");
|
||||
assertFalse(m.isReady());
|
||||
Mockito.verify(ep, Mockito.never()).enableEndpoint("form-detection");
|
||||
}
|
||||
|
||||
@Test
|
||||
void secondConcurrentInstallIsRejected(@TempDir Path dir) throws Exception {
|
||||
CountDownLatch gate = new CountDownLatch(1);
|
||||
server.createContext(
|
||||
"/gated.onnx",
|
||||
ex -> {
|
||||
try {
|
||||
gate.await(3, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
ex.sendResponseHeaders(200, modelBytes.length);
|
||||
ex.getResponseBody().write(modelBytes);
|
||||
ex.close();
|
||||
});
|
||||
FormDetectionModelManager m =
|
||||
manager(
|
||||
dir,
|
||||
entry("http://127.0.0.1:" + port + "/gated.onnx", modelSha),
|
||||
Mockito.mock(EndpointConfiguration.class));
|
||||
|
||||
m.startInstall("test-model"); // begins, blocks in handler
|
||||
// installing flag is set synchronously before the worker thread spawns
|
||||
assertThrows(IllegalStateException.class, () -> m.startInstall("test-model"));
|
||||
gate.countDown();
|
||||
awaitState(m, "ready", 5000);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsBlankUrl(@TempDir Path dir) {
|
||||
FormDetectionModelManager m =
|
||||
manager(dir, entry("", ""), Mockito.mock(EndpointConfiguration.class));
|
||||
assertThrows(IllegalStateException.class, () -> m.startInstall("test-model"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnknownModelId(@TempDir Path dir) {
|
||||
FormDetectionModelManager m =
|
||||
manager(
|
||||
dir,
|
||||
entry("http://127.0.0.1:" + port + "/model.onnx", modelSha),
|
||||
Mockito.mock(EndpointConfiguration.class));
|
||||
assertThrows(IllegalArgumentException.class, () -> m.startInstall("unknown"));
|
||||
}
|
||||
}
|
||||
+2
-53
@@ -13,9 +13,9 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@@ -52,58 +52,7 @@ class IntegrationConfigServiceTest {
|
||||
@Mock
|
||||
private stirling.software.proprietary.access.repository.ResourceGrantRepository grantRepository;
|
||||
|
||||
@Mock private IntegrationConfigValidator validator;
|
||||
@Mock private IntegrationConfigUsageCheck usageCheck;
|
||||
|
||||
private IntegrationConfigService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service =
|
||||
new IntegrationConfigService(
|
||||
repository,
|
||||
ownership,
|
||||
secretMasker,
|
||||
grantRepository,
|
||||
List.of(validator),
|
||||
List.of(usageCheck));
|
||||
}
|
||||
|
||||
@Test
|
||||
void createRejectsAConfigItsTypeValidatorRefuses() {
|
||||
when(secretMasker.sanitize(any())).thenReturn(Map.of());
|
||||
when(validator.type()).thenReturn(IntegrationType.API);
|
||||
org.mockito.Mockito.doThrow(new IllegalArgumentException("api config needs a 'url'"))
|
||||
.when(validator)
|
||||
.validate(any());
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
service.create(
|
||||
request(IntegrationType.API, OwnerScope.USER, null),
|
||||
user(7)))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.BAD_REQUEST));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteRefusedWhileAnythingStillReferencesTheConfig() {
|
||||
IntegrationConfig cfg = config(9L);
|
||||
when(repository.findById(9L)).thenReturn(Optional.of(cfg));
|
||||
when(ownership.canManage(any(), eq(cfg), any())).thenReturn(true);
|
||||
when(usageCheck.usagesOf(9L)).thenReturn(List.of("source 'Claims intake'"));
|
||||
|
||||
assertThatThrownBy(() -> service.delete(9L, user(7)))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.CONFLICT));
|
||||
verify(repository, org.mockito.Mockito.never()).delete(any(IntegrationConfig.class));
|
||||
}
|
||||
@InjectMocks private IntegrationConfigService service;
|
||||
|
||||
@Test
|
||||
void createDelegatesOwnershipAndSanitizesConfig() {
|
||||
|
||||
-24
@@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -195,29 +194,6 @@ class PolicyControllerTest {
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.BAD_REQUEST));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects an ad-hoc output the caller cannot use, on the request thread")
|
||||
void rejectsUnauthorizedAdHocOutput() {
|
||||
// The confused-deputy guard: an S3 output referencing a connection the caller may not
|
||||
// use is validated here (principal present) and refused before any worker dispatch.
|
||||
PipelineDefinition definition =
|
||||
new PipelineDefinition(
|
||||
"pipe",
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", null)),
|
||||
new OutputSpec("s3", Map.of("connectionId", 999)));
|
||||
doThrow(new IllegalArgumentException("unknown or inaccessible s3 connection"))
|
||||
.when(policyValidator)
|
||||
.validateOutput(any());
|
||||
|
||||
assertThatThrownBy(() -> controller.run(definition, new PolicyRunFiles()))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.BAD_REQUEST));
|
||||
verify(policyRunner, never()).runAdHoc(any(), any(), any());
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
|
||||
+2
-2
@@ -166,8 +166,8 @@ class PolicyExecutorTest {
|
||||
new PipelineStep(
|
||||
createPdf,
|
||||
Map.of(
|
||||
"document",
|
||||
"{\"title\":\"PO\",\"sections\":[]}",
|
||||
"htmlContent",
|
||||
"<p>hi</p>",
|
||||
"filename",
|
||||
"purchase-order.pdf"))),
|
||||
PolicyInputs.of(List.of()),
|
||||
|
||||
-22
@@ -82,28 +82,6 @@ class PolicyValidatorTest {
|
||||
assertTrue(ex.getMessage().contains("schedule"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateOutputDelegatesToTheSink() {
|
||||
when(outputSink.supports(any())).thenReturn(true);
|
||||
OutputSpec output = new OutputSpec("s3", Map.of("connectionId", 1));
|
||||
|
||||
validator.validateOutput(output);
|
||||
|
||||
verify(outputSink).validate(output);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateOutputSurfacesAnInaccessibleConnection() {
|
||||
when(outputSink.supports(any())).thenReturn(true);
|
||||
doThrow(new IllegalArgumentException("unknown or inaccessible s3 connection"))
|
||||
.when(outputSink)
|
||||
.validate(any());
|
||||
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> validator.validateOutput(new OutputSpec("s3", Map.of("connectionId", 1))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAnUnknownTriggerType() {
|
||||
when(trigger.type()).thenReturn("schedule");
|
||||
|
||||
+2
-7
@@ -23,7 +23,6 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
@@ -83,9 +82,7 @@ class S3InputSourceMinioTest {
|
||||
// The MinIO endpoint resolves to loopback, so the operator opt-in must be on.
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(true);
|
||||
source =
|
||||
new S3InputSource(
|
||||
new S3ConnectionPool(properties), S3TestConnections.legacyResolver());
|
||||
source = new S3InputSource(new S3ConnectionPool(properties));
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
}
|
||||
@@ -164,9 +161,7 @@ class S3InputSourceMinioTest {
|
||||
@Test
|
||||
void aPrivateEndpointIsRejectedWithoutTheOperatorOptIn() {
|
||||
S3InputSource guarded =
|
||||
new S3InputSource(
|
||||
new S3ConnectionPool(new ApplicationProperties()),
|
||||
S3TestConnections.legacyResolver());
|
||||
new S3InputSource(new S3ConnectionPool(new ApplicationProperties()));
|
||||
|
||||
assertThatThrownBy(() -> guarded.validate(spec(Map.of())))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
|
||||
+1
-3
@@ -29,7 +29,6 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.core.ResponseInputStream;
|
||||
import software.amazon.awssdk.core.exception.SdkClientException;
|
||||
@@ -65,8 +64,7 @@ class S3InputSourceTest {
|
||||
void setUp() {
|
||||
source =
|
||||
new S3InputSource(
|
||||
new S3ConnectionPool(new ApplicationProperties(), config -> s3Client),
|
||||
S3TestConnections.legacyResolver());
|
||||
new S3ConnectionPool(new ApplicationProperties(), config -> s3Client));
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
}
|
||||
|
||||
+2
-3
@@ -28,7 +28,6 @@ import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
@@ -91,8 +90,8 @@ class S3OutputSinkMinioTest {
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(true);
|
||||
S3ConnectionPool pool = new S3ConnectionPool(properties);
|
||||
ledger = new InProcessProcessedLedger();
|
||||
sink = new S3OutputSink(pool, S3TestConnections.legacyResolver(), ledger);
|
||||
source = new S3InputSource(pool, S3TestConnections.legacyResolver());
|
||||
sink = new S3OutputSink(pool, ledger);
|
||||
source = new S3InputSource(pool);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
-2
@@ -32,7 +32,6 @@ import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedFileStatus;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.awscore.exception.AwsServiceException;
|
||||
import software.amazon.awssdk.core.exception.SdkClientException;
|
||||
@@ -67,7 +66,6 @@ class S3OutputSinkTest {
|
||||
sink =
|
||||
new S3OutputSink(
|
||||
new S3ConnectionPool(new ApplicationProperties(), config -> s3Client),
|
||||
S3TestConnections.legacyResolver(),
|
||||
ledger);
|
||||
}
|
||||
|
||||
|
||||
-212
@@ -1,212 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.proprietary.access.model.OwnerScope;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
|
||||
/**
|
||||
* Tests for {@link EmbeddedS3CredentialMigration}: legacy embedded credentials become deduplicated
|
||||
* team-scoped connections, rewritten rows keep only per-use options, and re-runs are no-ops.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class EmbeddedS3CredentialMigrationTest {
|
||||
|
||||
@Mock private IntegrationConfigRepository connections;
|
||||
@Mock private TeamRepository teamRepository;
|
||||
|
||||
private final InProcessSourceStore sourceStore = new InProcessSourceStore();
|
||||
private final InProcessPolicyStore policyStore = new InProcessPolicyStore();
|
||||
private EmbeddedS3CredentialMigration migration;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
migration =
|
||||
new EmbeddedS3CredentialMigration(
|
||||
sourceStore, policyStore, connections, teamRepository);
|
||||
AtomicLong ids = new AtomicLong(100);
|
||||
// Lenient: the nothing-to-migrate cases never create a connection.
|
||||
lenient().when(connections.findAll()).thenReturn(List.of());
|
||||
lenient()
|
||||
.when(connections.save(any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
IntegrationConfig saved = invocation.getArgument(0);
|
||||
if (saved.getId() == null) {
|
||||
saved.setId(ids.incrementAndGet());
|
||||
}
|
||||
return saved;
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void extractsSharedCredentialsIntoOneTeamScopedConnection() {
|
||||
Team team = new Team();
|
||||
team.setId(7L);
|
||||
when(teamRepository.findById(7L)).thenReturn(Optional.of(team));
|
||||
Source source =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Claims intake",
|
||||
"s3",
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"prefix", "incoming/",
|
||||
"mode", "snapshot",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh"),
|
||||
true,
|
||||
"alice",
|
||||
7L));
|
||||
Policy policy =
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Rotate",
|
||||
"alice",
|
||||
true,
|
||||
null,
|
||||
List.of(source.id()),
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
new OutputSpec(
|
||||
"s3",
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"prefix", "processed/",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh")),
|
||||
7L));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
// Same bucket + credentials on both rows: exactly one connection extracted.
|
||||
verify(connections, times(1)).save(any());
|
||||
Map<String, Object> sourceOptions = sourceStore.get(source.id()).orElseThrow().options();
|
||||
assertEquals(101L, sourceOptions.get("connectionId"));
|
||||
assertEquals("incoming/", sourceOptions.get("prefix"));
|
||||
assertEquals("snapshot", sourceOptions.get("mode"));
|
||||
assertNull(sourceOptions.get("accessKeyId"));
|
||||
assertNull(sourceOptions.get("secretAccessKey"));
|
||||
assertNull(sourceOptions.get("bucket"));
|
||||
|
||||
Map<String, Object> outputOptions =
|
||||
policyStore.get(policy.id()).orElseThrow().output().options();
|
||||
assertEquals(101L, outputOptions.get("connectionId"));
|
||||
assertEquals("processed/", outputOptions.get("prefix"));
|
||||
assertNull(outputOptions.get("secretAccessKey"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void connectionOwnershipFollowsTheSourceTeam() {
|
||||
Team team = new Team();
|
||||
team.setId(7L);
|
||||
when(teamRepository.findById(7L)).thenReturn(Optional.of(team));
|
||||
sourceStore.save(s3Source("teamed", 7L));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
verify(connections)
|
||||
.save(
|
||||
org.mockito.ArgumentMatchers.argThat(
|
||||
connection ->
|
||||
connection.getScope() == OwnerScope.TEAM
|
||||
&& connection.getOwnerTeam() == team));
|
||||
}
|
||||
|
||||
@Test
|
||||
void teamlessRowsBecomeServerScopedConnections() {
|
||||
sourceStore.save(s3Source("solo", null));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
verify(connections)
|
||||
.save(
|
||||
org.mockito.ArgumentMatchers.argThat(
|
||||
connection -> connection.getScope() == OwnerScope.SERVER));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSecondRunFindsNothingToDo() {
|
||||
sourceStore.save(s3Source("once", null));
|
||||
|
||||
migration.migrate();
|
||||
migration.migrate();
|
||||
|
||||
// One connection from the first run; the rewritten source no longer embeds credentials.
|
||||
verify(connections, times(1)).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonS3AndAlreadyMigratedRowsAreUntouched() {
|
||||
Source folder =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Folder",
|
||||
"folder",
|
||||
Map.of("directory", "/in"),
|
||||
true,
|
||||
"alice",
|
||||
null));
|
||||
Source migrated =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Done already",
|
||||
"s3",
|
||||
Map.of("connectionId", 55L, "prefix", "in/"),
|
||||
true,
|
||||
"alice",
|
||||
null));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
verify(connections, times(0)).save(any());
|
||||
assertEquals(
|
||||
Map.of("directory", "/in"), sourceStore.get(folder.id()).orElseThrow().options());
|
||||
assertEquals(
|
||||
Map.of("connectionId", 55L, "prefix", "in/"),
|
||||
sourceStore.get(migrated.id()).orElseThrow().options());
|
||||
}
|
||||
|
||||
private static Source s3Source(String name, Long teamId) {
|
||||
return new Source(
|
||||
null,
|
||||
name,
|
||||
"s3",
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh"),
|
||||
true,
|
||||
"alice",
|
||||
teamId);
|
||||
}
|
||||
}
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
|
||||
|
||||
/** Tests for {@link PolicyS3ConnectionUsageCheck}'s reference scan across sources and outputs. */
|
||||
class PolicyS3ConnectionUsageCheckTest {
|
||||
|
||||
private final InProcessSourceStore sourceStore = new InProcessSourceStore();
|
||||
private final InProcessPolicyStore policyStore = new InProcessPolicyStore();
|
||||
private final PolicyS3ConnectionUsageCheck check =
|
||||
new PolicyS3ConnectionUsageCheck(sourceStore, policyStore);
|
||||
|
||||
@Test
|
||||
void reportsSourcesAndOutputsReferencingTheConnection() {
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Claims intake",
|
||||
"s3",
|
||||
Map.of("connectionId", 5L, "prefix", "in/"),
|
||||
true,
|
||||
"alice",
|
||||
null));
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Rotate",
|
||||
"alice",
|
||||
true,
|
||||
null,
|
||||
List.of(),
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
new OutputSpec("s3", Map.of("connectionId", "5")),
|
||||
null));
|
||||
|
||||
assertThat(check.usagesOf(5))
|
||||
.containsExactlyInAnyOrder("source 'Claims intake'", "pipeline 'Rotate'");
|
||||
assertThat(check.usagesOf(6)).isEmpty();
|
||||
}
|
||||
}
|
||||
-149
@@ -1,149 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/**
|
||||
* Tests for {@link S3ConnectionResolver}: connection dereferencing with per-use overrides, the
|
||||
* legacy embedded fallback, and the save-time access check that background sweeps skip.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class S3ConnectionResolverTest {
|
||||
|
||||
@Mock private IntegrationConfigRepository connections;
|
||||
@Mock private OwnershipService ownership;
|
||||
@Mock private UserService userService;
|
||||
|
||||
@AfterEach
|
||||
void clearSecurityContext() {
|
||||
SecurityContextHolder.clearContext();
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvesAConnectionAndMergesPerUseOptions() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
|
||||
S3Config config =
|
||||
resolver()
|
||||
.resolve(
|
||||
Map.of(
|
||||
"connectionId", 9L,
|
||||
"prefix", "incoming/",
|
||||
"mode", "snapshot"));
|
||||
|
||||
assertEquals("inbox", config.bucket());
|
||||
assertEquals("AKIAEXAMPLE", config.accessKeyId());
|
||||
assertEquals("incoming/", config.prefix());
|
||||
assertTrue(config.snapshot());
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsAStringConnectionReference() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
|
||||
assertEquals("inbox", resolver().resolve(Map.of("connectionId", "9")).bucket());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallsBackToLegacyEmbeddedCredentials() {
|
||||
S3Config config =
|
||||
resolver()
|
||||
.resolve(
|
||||
Map.of(
|
||||
"bucket", "legacy",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh"));
|
||||
|
||||
assertEquals("legacy", config.bucket());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnknownDisabledOrWrongTypeConnections() {
|
||||
when(connections.findById(1L)).thenReturn(Optional.empty());
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 1L)));
|
||||
|
||||
when(connections.findById(2L)).thenReturn(Optional.of(s3Connection(2L, false)));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 2L)));
|
||||
|
||||
IntegrationConfig mcp = s3Connection(3L, true);
|
||||
mcp.setIntegrationType(IntegrationType.MCP);
|
||||
when(connections.findById(3L)).thenReturn(Optional.of(mcp));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 3L)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAuthenticatedSaverMustBeAllowedToUseTheConnection() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
User saver = new User();
|
||||
saver.setUsername("alice");
|
||||
SecurityContextHolder.getContext()
|
||||
.setAuthentication(
|
||||
new UsernamePasswordAuthenticationToken(saver, null, java.util.List.of()));
|
||||
when(ownership.canUse(any(), any(IntegrationConfig.class), eq(saver))).thenReturn(false);
|
||||
|
||||
// Denied reads the same as unknown and never echoes the connection name, so ids can't be
|
||||
// enumerated by probing.
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 9L)));
|
||||
try {
|
||||
resolver().resolve(Map.of("connectionId", 9L));
|
||||
} catch (IllegalArgumentException e) {
|
||||
org.junit.jupiter.api.Assertions.assertFalse(
|
||||
e.getMessage().contains("Claims bucket"),
|
||||
"access-denied error must not leak the connection name");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void backgroundSweepsWithNoUserSkipTheAccessCheck() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
|
||||
// No authentication in the context: resolution succeeds without consulting ownership.
|
||||
assertEquals("inbox", resolver().resolve(Map.of("connectionId", 9L)).bucket());
|
||||
}
|
||||
|
||||
private S3ConnectionResolver resolver() {
|
||||
return new S3ConnectionResolver(connections, ownership, userService);
|
||||
}
|
||||
|
||||
private static IntegrationConfig s3Connection(long id, boolean enabled) {
|
||||
IntegrationConfig connection = new IntegrationConfig();
|
||||
connection.setId(id);
|
||||
connection.setIntegrationType(IntegrationType.S3);
|
||||
connection.setName("Claims bucket");
|
||||
connection.setEnabled(enabled);
|
||||
connection.setConfig(
|
||||
"{\"bucket\":\"inbox\",\"accessKeyId\":\"AKIAEXAMPLE\","
|
||||
+ "\"secretAccessKey\":\"shh\"}");
|
||||
return connection;
|
||||
}
|
||||
}
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
|
||||
/**
|
||||
* Tests for {@link S3IntegrationValidator}: the S3 connection schema fails at save time - missing
|
||||
* credentials, bad endpoints, and private endpoints without the operator opt-in.
|
||||
*/
|
||||
class S3IntegrationValidatorTest {
|
||||
|
||||
@Test
|
||||
void acceptsACompleteConnection() {
|
||||
assertThatCode(
|
||||
() ->
|
||||
validator(false)
|
||||
.validate(
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh")))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsMissingCredentialsOrBucket() {
|
||||
assertThatThrownBy(() -> validator(false).validate(Map.of("bucket", "inbox")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
validator(false)
|
||||
.validate(
|
||||
Map.of(
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAPrivateEndpointWithoutTheOperatorOptIn() {
|
||||
Map<String, Object> config =
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh",
|
||||
"endpoint", "http://localhost:9000");
|
||||
|
||||
assertThatThrownBy(() -> validator(false).validate(config))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("allowPrivateS3Endpoints");
|
||||
assertThatCode(() -> validator(true).validate(config)).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void itOnlyClaimsTheS3Type() {
|
||||
org.junit.jupiter.api.Assertions.assertEquals(IntegrationType.S3, validator(false).type());
|
||||
}
|
||||
|
||||
private static S3IntegrationValidator validator(boolean allowPrivateEndpoints) {
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(allowPrivateEndpoints);
|
||||
return new S3IntegrationValidator(properties);
|
||||
}
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/** Test fixtures for S3 connection plumbing shared across the policy S3 tests. */
|
||||
public final class S3TestConnections {
|
||||
|
||||
private S3TestConnections() {}
|
||||
|
||||
/**
|
||||
* A resolver for tests whose options embed credentials directly (the legacy pass-through path),
|
||||
* so its collaborators are never touched.
|
||||
*/
|
||||
public static S3ConnectionResolver legacyResolver() {
|
||||
return new S3ConnectionResolver(
|
||||
mock(IntegrationConfigRepository.class),
|
||||
mock(OwnershipService.class),
|
||||
mock(UserService.class));
|
||||
}
|
||||
}
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
package stirling.software.proprietary.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.model.api.ai.create.AiDocument;
|
||||
|
||||
class AiDocumentHtmlRendererTest {
|
||||
|
||||
private final AiDocumentHtmlRenderer renderer = new AiDocumentHtmlRenderer();
|
||||
|
||||
private static AiDocument.Section section(String type) {
|
||||
AiDocument.Section s = new AiDocument.Section();
|
||||
s.setType(type);
|
||||
return s;
|
||||
}
|
||||
|
||||
private static AiDocument document(String title, List<AiDocument.Section> sections) {
|
||||
AiDocument doc = new AiDocument();
|
||||
doc.setTitle(title);
|
||||
doc.setSections(sections);
|
||||
return doc;
|
||||
}
|
||||
|
||||
@Test
|
||||
void rendersAllSectionTypes() {
|
||||
AiDocument.Section text = section("text");
|
||||
text.setBody("Some prose text.");
|
||||
AiDocument.Section kv = section("key_value");
|
||||
kv.setPairs(List.of(List.of("Key", "Value")));
|
||||
AiDocument.Section items = section("line_items");
|
||||
items.setColumns(List.of("A", "B"));
|
||||
items.setRows(List.of(List.of("1", "2")));
|
||||
AiDocument.Section bullets = section("bullet_list");
|
||||
bullets.setItems(List.of("item one"));
|
||||
AiDocument.Section sign = section("signature");
|
||||
sign.setSignatories(List.of("Alice"));
|
||||
|
||||
String html = renderer.render(document("All", List.of(text, kv, items, bullets, sign)));
|
||||
|
||||
assertTrue(html.contains("<!DOCTYPE html>"));
|
||||
assertTrue(html.contains("Some prose text."));
|
||||
assertTrue(html.contains("Key") && html.contains("Value"));
|
||||
assertTrue(html.contains("<th>"));
|
||||
assertTrue(html.contains("item one"));
|
||||
assertTrue(html.contains("Alice"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rendersMarkupCharactersAsText() {
|
||||
AiDocument.Section text = section("text");
|
||||
text.setBody("a <b>x</b> & y");
|
||||
|
||||
String html = renderer.render(document("Doc", List.of(text)));
|
||||
|
||||
assertFalse(html.contains("<b>"));
|
||||
assertTrue(html.contains("<b>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void totalRowRenderedWhenPresent() {
|
||||
AiDocument.Section items = section("line_items");
|
||||
items.setColumns(List.of("Item", "Total"));
|
||||
items.setRows(List.of(List.of("Widget", "$10")));
|
||||
items.setTotalRow(List.of("Total", "$10"));
|
||||
|
||||
assertTrue(
|
||||
renderer.render(document("Table", List.of(items)))
|
||||
.contains("<tr class=\"total-row\">"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void totalRowAbsentWhenNotProvided() {
|
||||
AiDocument.Section items = section("line_items");
|
||||
items.setColumns(List.of("Item"));
|
||||
items.setRows(List.of(List.of("Widget")));
|
||||
|
||||
assertFalse(
|
||||
renderer.render(document("Table", List.of(items)))
|
||||
.contains("<tr class=\"total-row\">"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rendersSubtitleAndReference() {
|
||||
AiDocument doc = document("My Doc", List.of());
|
||||
doc.setSubtitle("Subtitle Here");
|
||||
doc.setReferenceNumber("REF-42");
|
||||
|
||||
String html = renderer.render(doc);
|
||||
|
||||
assertTrue(html.contains("Subtitle Here"));
|
||||
assertTrue(html.contains("REF-42"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesHexColourOverride() {
|
||||
AiDocument doc = document("Styled", List.of());
|
||||
AiDocument.Style style = new AiDocument.Style();
|
||||
style.setPrimaryColor("#ff00ff");
|
||||
style.setBackgroundColor("#111111");
|
||||
doc.setStyle(style);
|
||||
|
||||
String html = renderer.render(doc);
|
||||
|
||||
assertTrue(html.contains("--color-primary: #ff00ff"));
|
||||
assertTrue(html.contains("--color-bg: #111111"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoresColourWithDisallowedCharacters() {
|
||||
AiDocument doc = document("Styled", List.of());
|
||||
AiDocument.Style style = new AiDocument.Style();
|
||||
style.setPrimaryColor("rgb(255, 0, 0)");
|
||||
doc.setStyle(style);
|
||||
|
||||
String html = renderer.render(doc);
|
||||
|
||||
assertFalse(html.contains("rgb("));
|
||||
assertTrue(html.contains("<!DOCTYPE html>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoresNonHexColour() {
|
||||
AiDocument doc = document("Styled", List.of());
|
||||
AiDocument.Style style = new AiDocument.Style();
|
||||
style.setPrimaryColor("magenta");
|
||||
style.setBackgroundColor("#fff");
|
||||
doc.setStyle(style);
|
||||
|
||||
String html = renderer.render(doc);
|
||||
|
||||
assertFalse(html.contains("--color-primary: magenta"));
|
||||
assertFalse(html.contains("--color-bg: #fff;"));
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
-- Classification labels are now a fixed, built-in set bundled with the app and sent to the engine
|
||||
-- per request (see ClassificationLabelProvider); the per-team classification_labels table (created
|
||||
-- in V30) is no longer read or written. Drop it.
|
||||
--
|
||||
-- Forward migration: V30 is kept so any DB that already applied it still validates. This runs after
|
||||
-- V30 in every case, so it drops the table whether V30 just created it (fresh DB) or it was created
|
||||
-- and populated on an earlier deploy. IF EXISTS only guards the edge case where the table is already
|
||||
-- absent, keeping the migration safe to apply regardless of prior state.
|
||||
|
||||
DROP TABLE IF EXISTS classification_labels;
|
||||
+1
-10
@@ -36,8 +36,6 @@ ext {
|
||||
okhttpBomVersion = "5.3.2"
|
||||
gsonVersion = "2.14.0"
|
||||
guavaVersion = "33.6.0-jre"
|
||||
jinjavaVersion = "2.8.3"
|
||||
jackson2Version = "2.21.2"
|
||||
bucket4jVersion = "8.19.0"
|
||||
archunitVersion = "1.4.2"
|
||||
batikVersion = "1.19"
|
||||
@@ -93,7 +91,7 @@ springBoot {
|
||||
|
||||
allprojects {
|
||||
group = 'stirling.software'
|
||||
version = '2.14.2'
|
||||
version = '2.14.1'
|
||||
|
||||
configurations.configureEach {
|
||||
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
|
||||
@@ -224,13 +222,6 @@ subprojects {
|
||||
resolutionStrategy.force "org.apache.commons:commons-lang3:${commonsLang3}"
|
||||
// CVE-2024-47554: commons-io DoS prevention
|
||||
resolutionStrategy.force "commons-io:commons-io:${commonsIoVersion}"
|
||||
// Jackson 2 is transitive-only here (jinjava, opensaml, jjwt request older versions);
|
||||
// pin the family to a current release and keep modules aligned.
|
||||
resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson2Version}"
|
||||
resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson2Version}"
|
||||
resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson2Version}"
|
||||
resolutionStrategy.force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2Version}"
|
||||
resolutionStrategy.force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2Version}"
|
||||
// Keep BouncyCastle modules aligned to avoid runtime linkage errors
|
||||
resolutionStrategy.force "org.bouncycastle:bcprov-jdk18on:${bouncycastleVersion}"
|
||||
resolutionStrategy.force "org.bouncycastle:bcpkix-jdk18on:${bouncycastleVersion}"
|
||||
|
||||
@@ -19,6 +19,15 @@ RUN apt-get update \
|
||||
&& rm /tmp/task.deb \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Pre-download the default Auto Form Detection model (FFDNet-S, ~37MB) so the feature works
|
||||
# out-of-the-box. Verified by checksum here; seeded into the writable configs model dir at startup
|
||||
# by FormDetectionModelManager (FORMDETECTION_PREINSTALLEDMODELDIR below).
|
||||
ARG FORM_DETECTION_MODEL_URL=https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/main/FFDNet-S.onnx
|
||||
ARG FORM_DETECTION_MODEL_SHA256=93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0
|
||||
RUN mkdir -p /preinstalled-models \
|
||||
&& curl -fSL "${FORM_DETECTION_MODEL_URL}" -o /preinstalled-models/ffdnet-s.onnx \
|
||||
&& echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/ffdnet-s.onnx" | sha256sum -c -
|
||||
|
||||
# JDK 25+: --add-exports is no longer accepted via JAVA_TOOL_OPTIONS; use JDK_JAVA_OPTIONS instead
|
||||
ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
||||
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
||||
@@ -55,6 +64,7 @@ RUN JPDFIUM_PLATFORM="$([ "$TARGETARCH" = arm64 ] && echo linux-arm64 || echo li
|
||||
-PbuildWithPortal=${BUILD_PORTAL} \
|
||||
-PjpdfiumPlatforms="$JPDFIUM_PLATFORM" \
|
||||
-PprototypesMode=${PROTOTYPES_BUILD} \
|
||||
-PbundleOnnxRuntime=true \
|
||||
-x spotlessApply -x spotlessCheck -x test -x sonarqube \
|
||||
--no-daemon
|
||||
|
||||
@@ -64,6 +74,15 @@ WORKDIR /tmp
|
||||
COPY --from=app-build /app/app/core/build/libs/*.jar app.jar
|
||||
RUN java -Djarmode=tools -jar app.jar extract --layers --destination /layers
|
||||
|
||||
# Slim the bundled ONNX Runtime (included only via -PbundleOnnxRuntime above) to the target Linux
|
||||
# arch (~42MB jar -> ~8MB). Docker server image only; desktop/local builds don't bundle it at all.
|
||||
# The detection model itself is unaffected - it is still downloaded on demand at runtime.
|
||||
COPY scripts/slim-onnxruntime.sh /tmp/slim-onnxruntime.sh
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends zip \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& sh /tmp/slim-onnxruntime.sh /layers/dependencies
|
||||
|
||||
|
||||
# Stage 3: Final runtime image on top of pre-built base
|
||||
FROM ${BASE_IMAGE}
|
||||
@@ -82,6 +101,9 @@ COPY --link --from=app-build --chown=1000:1000 \
|
||||
/app/build/libs/restart-helper.jar /restart-helper.jar
|
||||
COPY --link --chown=1000:1000 scripts/ /scripts/
|
||||
|
||||
# Pre-bundled default detection model; FormDetectionModelManager seeds it into /configs on startup.
|
||||
COPY --link --from=app-build --chown=1000:1000 /preinstalled-models/ /opt/stirling/preinstalled-models/
|
||||
|
||||
# Fonts go to system dir, root ownership is correct (world-readable)
|
||||
COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/truetype/
|
||||
|
||||
@@ -105,6 +127,7 @@ RUN echo "${VERSION_TAG:-dev}" > /etc/stirling_version
|
||||
|
||||
# Environment variables
|
||||
ENV VERSION_TAG=$VERSION_TAG \
|
||||
FORMDETECTION_PREINSTALLEDMODELDIR="/opt/stirling/preinstalled-models" \
|
||||
STIRLING_AOT_ENABLE="false" \
|
||||
STIRLING_JVM_PROFILE="balanced" \
|
||||
_JVM_OPTS_BALANCED="-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/configs/heap_dumps -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=4m -XX:G1PeriodicGCInterval=60000 -XX:+UseStringDeduplication -XX:+UseCompactObjectHeaders -XX:+ExplicitGCInvokesConcurrent -Dspring.threads.virtual.enabled=true -Djava.awt.headless=true" \
|
||||
|
||||
@@ -20,6 +20,15 @@ RUN apt-get update \
|
||||
&& rm /tmp/task.deb \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Pre-download the default Auto Form Detection model (FFDNet-S, ~37MB) so the feature works
|
||||
# out-of-the-box - especially important for this air-gapped fat image (no runtime internet).
|
||||
# Verified by checksum; seeded into the writable configs model dir at startup.
|
||||
ARG FORM_DETECTION_MODEL_URL=https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/main/FFDNet-S.onnx
|
||||
ARG FORM_DETECTION_MODEL_SHA256=93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0
|
||||
RUN mkdir -p /preinstalled-models \
|
||||
&& curl -fSL "${FORM_DETECTION_MODEL_URL}" -o /preinstalled-models/ffdnet-s.onnx \
|
||||
&& echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/ffdnet-s.onnx" | sha256sum -c -
|
||||
|
||||
# JDK 25+: --add-exports is no longer accepted via JAVA_TOOL_OPTIONS; use JDK_JAVA_OPTIONS instead
|
||||
ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
||||
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
||||
@@ -48,6 +57,7 @@ RUN JPDFIUM_PLATFORM="$([ "$TARGETARCH" = arm64 ] && echo linux-arm64 || echo li
|
||||
DISABLE_ADDITIONAL_FEATURES=false \
|
||||
gradle clean build \
|
||||
-PbuildWithFrontend=true \
|
||||
-PbundleOnnxRuntime=true \
|
||||
-PbuildWithPortal=${BUILD_PORTAL} \
|
||||
-PjpdfiumPlatforms="$JPDFIUM_PLATFORM" \
|
||||
-x spotlessApply -x spotlessCheck -x test -x sonarqube \
|
||||
@@ -59,6 +69,14 @@ WORKDIR /tmp
|
||||
COPY --from=app-build /app/app/core/build/libs/*.jar app.jar
|
||||
RUN java -Djarmode=tools -jar app.jar extract --layers --destination /layers
|
||||
|
||||
# Slim the bundled ONNX Runtime (included only via -PbundleOnnxRuntime above) to the target Linux
|
||||
# arch (~42MB jar -> ~8MB). Docker server image only; desktop/local builds don't bundle it at all.
|
||||
COPY scripts/slim-onnxruntime.sh /tmp/slim-onnxruntime.sh
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends zip \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& sh /tmp/slim-onnxruntime.sh /layers/dependencies
|
||||
|
||||
|
||||
# Stage 3: Final runtime image on top of pre-built base
|
||||
FROM ${BASE_IMAGE}
|
||||
@@ -77,6 +95,9 @@ COPY --link --from=app-build --chown=1000:1000 \
|
||||
/app/build/libs/restart-helper.jar /restart-helper.jar
|
||||
COPY --link --chown=1000:1000 scripts/ /scripts/
|
||||
|
||||
# Pre-bundled default detection model; FormDetectionModelManager seeds it into /configs on startup.
|
||||
COPY --link --from=app-build --chown=1000:1000 /preinstalled-models/ /opt/stirling/preinstalled-models/
|
||||
|
||||
# Fonts go to system dir, root ownership is correct (world-readable)
|
||||
COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/truetype/
|
||||
|
||||
@@ -99,6 +120,7 @@ RUN echo "${VERSION_TAG:-dev}" > /etc/stirling_version
|
||||
|
||||
# Environment variables
|
||||
ENV VERSION_TAG=$VERSION_TAG \
|
||||
FORMDETECTION_PREINSTALLEDMODELDIR="/opt/stirling/preinstalled-models" \
|
||||
STIRLING_AOT_ENABLE="false" \
|
||||
STIRLING_JVM_PROFILE="balanced" \
|
||||
_JVM_OPTS_BALANCED="-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/configs/heap_dumps -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=4m -XX:G1PeriodicGCInterval=60000 -XX:+UseStringDeduplication -XX:+UseCompactObjectHeaders -XX:+ExplicitGCInvokesConcurrent -Dspring.threads.virtual.enabled=true -Djava.awt.headless=true" \
|
||||
|
||||
@@ -5,6 +5,7 @@ description = "AI Document Engine"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"fastapi>=0.116.0",
|
||||
"jinja2>=3.1.0",
|
||||
"pgvector>=0.3.6",
|
||||
"psycopg[binary,pool]>=3.2",
|
||||
"pydantic>=2.0.0",
|
||||
|
||||
@@ -116,8 +116,8 @@ class DocumentClassifierAgent:
|
||||
)
|
||||
|
||||
async def classify(self, request: ClassifyDocumentRequest) -> ClassifyDocumentResponse:
|
||||
# The caller (the backend) always supplies the allowed vocabulary — its
|
||||
# fixed built-in label set — so the engine holds no vocabulary of its own.
|
||||
# The caller (the backend) always supplies the allowed vocabulary — the
|
||||
# team's stored labels — so the engine holds no vocabulary of its own.
|
||||
allowed = request.labels
|
||||
window = select_window(request.pages)
|
||||
prompt = self._build_prompt(request.file_name, allowed, window)
|
||||
|
||||
@@ -10,7 +10,7 @@ Flow:
|
||||
4. SectionWriterAgents (smart_model) run in parallel via asyncio.gather.
|
||||
Each returns a WrittenSections with fully populated DocumentSection objects.
|
||||
5. The assembler collects sections in plan order → GeneratedDocument.
|
||||
6. The assembled document is emitted as structured fields. The LLM never writes HTML.
|
||||
6. Jinja renders the document to HTML. The LLM never writes HTML.
|
||||
|
||||
The planner is split into two calls (meta then sections) so each LLM output schema
|
||||
stays small enough for grammar compilation on all model tiers including Haiku.
|
||||
@@ -22,7 +22,9 @@ import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_ai.output import NativeOutput
|
||||
|
||||
@@ -49,6 +51,8 @@ from stirling.services import AppRuntime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TEMPLATES_DIR = Path(__file__).parent / "templates"
|
||||
|
||||
# ── Token budget ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Conservative per-section token estimates mapped from planner-assigned depth.
|
||||
@@ -162,13 +166,10 @@ Analyse the user's request and produce a DocumentMeta with:
|
||||
document, if the user provides one. Leave empty if the user provides no such context.
|
||||
|
||||
- style_primary_color: accent and heading colour. Set ONLY when the user explicitly names a
|
||||
colour or colour scheme (e.g. "make it red", "use navy blue"). Express it as a 6-digit hex
|
||||
code in #RRGGBB format (map any named colour to its hex value yourself, e.g. "navy" →
|
||||
"#000080"). No other format is accepted. Leave null if no colour is stated.
|
||||
- style_background_color: page background colour, same #RRGGBB format. Set only if explicitly
|
||||
requested.
|
||||
- style_body_text_color: body text colour, same #RRGGBB format. Set only if explicitly
|
||||
requested.
|
||||
colour or colour scheme (e.g. "make it red", "use navy blue"). Use CSS named colours
|
||||
(e.g. "magenta", "navy", "crimson") or hex values. Leave null if no colour is stated.
|
||||
- style_background_color: page background colour. Set only if explicitly requested.
|
||||
- style_body_text_color: body text colour. Set only if explicitly requested.
|
||||
|
||||
- cannot_do_reason: set this ONLY when the request is not asking to create a document at all
|
||||
(e.g. a question, a greeting, an edit request to an existing document). Never set it
|
||||
@@ -298,6 +299,15 @@ def _build_writer_prompt(plan: DocumentPlan, chunk: _Chunk) -> str:
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _build_jinja_env() -> Environment:
|
||||
return Environment(
|
||||
loader=FileSystemLoader(str(_TEMPLATES_DIR)),
|
||||
autoescape=True,
|
||||
trim_blocks=True,
|
||||
lstrip_blocks=True,
|
||||
)
|
||||
|
||||
|
||||
def _safe_filename(title: str) -> str:
|
||||
slug = re.sub(r"[^\w\s-]", "", title.lower())
|
||||
slug = re.sub(r"[\s_-]+", "-", slug).strip("-")
|
||||
@@ -310,6 +320,7 @@ def _safe_filename(title: str) -> str:
|
||||
class PdfCreateAgent:
|
||||
def __init__(self, runtime: AppRuntime) -> None:
|
||||
self.runtime = runtime
|
||||
self._jinja_env = _build_jinja_env()
|
||||
|
||||
self._meta_planner: Agent[None, DocumentMeta] = Agent(
|
||||
model=runtime.smart_model,
|
||||
@@ -390,12 +401,14 @@ class PdfCreateAgent:
|
||||
sections=all_sections,
|
||||
)
|
||||
|
||||
# ── Phase 6: emit ──────────────────────────────────────────────────────
|
||||
# ── Phase 6: render ────────────────────────────────────────────────────
|
||||
logger.info("[pdf-create] phase 6/6: rendering HTML")
|
||||
html = self._render(doc)
|
||||
filename = _safe_filename(plan.title)
|
||||
logger.info(
|
||||
"[pdf-create] done — filename=%r sections=%d",
|
||||
"[pdf-create] done — filename=%r html_bytes=%d",
|
||||
filename,
|
||||
len(all_sections),
|
||||
len(html),
|
||||
)
|
||||
|
||||
return EditPlanResponse(
|
||||
@@ -404,7 +417,7 @@ class PdfCreateAgent:
|
||||
ToolOperationStep(
|
||||
tool=AgentToolId.CREATE_PDF_FROM_HTML_AGENT,
|
||||
parameters=CreatePdfFromHtmlAgentParams(
|
||||
document=doc.model_dump_json(),
|
||||
html_content=html,
|
||||
filename=filename,
|
||||
),
|
||||
)
|
||||
@@ -424,3 +437,7 @@ class PdfCreateAgent:
|
||||
len(result.output.sections),
|
||||
)
|
||||
return result.output
|
||||
|
||||
def _render(self, doc: GeneratedDocument) -> str:
|
||||
template = self._jinja_env.get_template("document.html.jinja2")
|
||||
return template.render(doc=doc)
|
||||
|
||||
+19
-21
@@ -1,4 +1,3 @@
|
||||
{%- autoescape true -%}
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
@@ -176,18 +175,18 @@
|
||||
color: var(--color-label);
|
||||
}
|
||||
</style>
|
||||
{%- if style_primary or style_background or style_body %}
|
||||
{%- if doc.style %}
|
||||
<style>
|
||||
:root {
|
||||
{%- if style_primary %}
|
||||
--color-primary: {{ style_primary }};
|
||||
{%- if doc.style.primary_color %}
|
||||
--color-primary: {{ doc.style.primary_color }};
|
||||
{%- endif %}
|
||||
{%- if style_background %}
|
||||
--color-bg: {{ style_background }};
|
||||
{%- if doc.style.background_color %}
|
||||
--color-bg: {{ doc.style.background_color }};
|
||||
{%- endif %}
|
||||
{%- if style_body %}
|
||||
--color-body: {{ style_body }};
|
||||
--color-label: {{ style_body }};
|
||||
{%- if doc.style.body_text_color %}
|
||||
--color-body: {{ doc.style.body_text_color }};
|
||||
--color-label: {{ doc.style.body_text_color }};
|
||||
{%- endif %}
|
||||
}
|
||||
</style>
|
||||
@@ -196,16 +195,16 @@
|
||||
<body>
|
||||
|
||||
<div class="doc-header">
|
||||
<div class="doc-title">{{ title }}</div>
|
||||
{%- if subtitle %}
|
||||
<div class="doc-subtitle">{{ subtitle }}</div>
|
||||
<div class="doc-title">{{ doc.title }}</div>
|
||||
{%- if doc.subtitle %}
|
||||
<div class="doc-subtitle">{{ doc.subtitle }}</div>
|
||||
{%- endif %}
|
||||
{%- if reference_number %}
|
||||
<div class="doc-reference">{{ reference_number }}</div>
|
||||
{%- if doc.reference_number %}
|
||||
<div class="doc-reference">{{ doc.reference_number }}</div>
|
||||
{%- endif %}
|
||||
</div>
|
||||
|
||||
{%- for section in sections %}
|
||||
{%- for section in doc.sections %}
|
||||
|
||||
{%- if section.type == "text" %}
|
||||
<section>
|
||||
@@ -213,8 +212,8 @@
|
||||
<h2>{{ section.heading }}</h2>
|
||||
{%- endif %}
|
||||
<div class="text-body">
|
||||
{%- for para in section.paragraphs %}
|
||||
<p>{{ para }}</p>
|
||||
{%- for para in section.body.split('\n\n') %}
|
||||
<p>{{ para | replace('\n', ' ') }}</p>
|
||||
{%- endfor %}
|
||||
</div>
|
||||
</section>
|
||||
@@ -226,10 +225,10 @@
|
||||
{%- endif %}
|
||||
<table class="kv-table">
|
||||
<tbody>
|
||||
{%- for pair in section.pairs %}
|
||||
{%- for label, value in section.pairs %}
|
||||
<tr>
|
||||
<td class="kv-label">{{ pair.label }}</td>
|
||||
<td class="kv-value">{{ pair.value }}</td>
|
||||
<td class="kv-label">{{ label }}</td>
|
||||
<td class="kv-value">{{ value }}</td>
|
||||
</tr>
|
||||
{%- endfor %}
|
||||
</tbody>
|
||||
@@ -300,4 +299,3 @@
|
||||
|
||||
</body>
|
||||
</html>
|
||||
{%- endautoescape %}
|
||||
@@ -1,14 +1,14 @@
|
||||
"""Contracts for the PDF Create Agent.
|
||||
|
||||
The agent accepts a natural-language prompt and returns a single
|
||||
CREATE_PDF_FROM_HTML_AGENT plan step carrying the assembled document.
|
||||
CREATE_PDF_FROM_HTML_AGENT plan step carrying the rendered HTML.
|
||||
|
||||
Pipeline:
|
||||
1. PlannerAgent (smart_model) → DocumentPlan: structured skeleton, no body text.
|
||||
2. Python chunks the plan by token budget.
|
||||
3. SectionWriterAgents (smart_model, parallel) → WrittenSections per chunk.
|
||||
4. Assembler collects sections in plan order → GeneratedDocument.
|
||||
5. The document is emitted as structured fields. The LLM never writes HTML.
|
||||
5. Jinja renders GeneratedDocument → HTML. The LLM never writes HTML.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -81,12 +81,14 @@ type DocumentSection = Annotated[
|
||||
]
|
||||
|
||||
|
||||
# Colours must be a 6-digit hex code (#RRGGBB); anything else is dropped to None.
|
||||
_SAFE_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{6}$")
|
||||
# Named colour or hex only — anything else is dropped so a colour can't inject CSS into the
|
||||
# <style> block (which would let WeasyPrint fetch an attacker-controlled url() → SSRF).
|
||||
_SAFE_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{3,8}$|^[a-zA-Z]{1,30}$")
|
||||
|
||||
|
||||
class DocumentStyle(ApiModel):
|
||||
"""Document colours, inferred by the meta planner. Non-hex values are dropped to ``None``."""
|
||||
"""Document colours, inferred by the meta planner and rendered into the engine's Jinja
|
||||
template (never sent to Java). Unsafe colours are dropped to ``None``."""
|
||||
|
||||
primary_color: str | None = Field(default=None)
|
||||
background_color: str | None = Field(default=None)
|
||||
@@ -101,7 +103,7 @@ class DocumentStyle(ApiModel):
|
||||
|
||||
|
||||
class GeneratedDocument(ApiModel):
|
||||
"""The full document model emitted for rendering."""
|
||||
"""The full document model passed to Jinja for HTML rendering."""
|
||||
|
||||
title: str
|
||||
subtitle: str | None = None
|
||||
|
||||
@@ -29,7 +29,7 @@ class PdfCommentAgentParams(ApiModel):
|
||||
|
||||
|
||||
class CreatePdfFromHtmlAgentParams(ApiModel):
|
||||
document: str
|
||||
html_content: str
|
||||
filename: str = Field(pattern=r"^.+\.pdf$")
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Coverage:
|
||||
1. Section model validation (each section type round-trips correctly)
|
||||
2. orchestrate() emits the assembled document as structured JSON
|
||||
2. Jinja rendering (_render produces valid HTML for each section type)
|
||||
3. _safe_filename produces clean slugs
|
||||
4. _make_chunks groups sections correctly by token budget
|
||||
5. orchestrate() produces the correct EditPlanResponse via planner + writer mocks
|
||||
@@ -12,8 +12,6 @@ Coverage:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from conftest import build_app_settings
|
||||
from pydantic_ai.models.test import TestModel
|
||||
@@ -66,6 +64,37 @@ def agent(runtime: AppRuntime) -> PdfCreateAgent:
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _invoice_doc() -> GeneratedDocument:
|
||||
return GeneratedDocument(
|
||||
title="Invoice",
|
||||
subtitle="Acme Corp",
|
||||
reference_number="Invoice #INV-001",
|
||||
sections=[
|
||||
KeyValueSection(
|
||||
heading="Details",
|
||||
pairs=[("Date", "2026-05-06"), ("Due", "2026-06-06"), ("Currency", "USD")],
|
||||
),
|
||||
LineItemsSection(
|
||||
heading="Line Items",
|
||||
columns=["Description", "Qty", "Unit Price", "Total"],
|
||||
rows=[
|
||||
["Consulting services", "10", "$500.00", "$5,000.00"],
|
||||
["Expenses", "1", "$200.00", "$200.00"],
|
||||
],
|
||||
total_row=["Total", "", "", "$5,200.00"],
|
||||
),
|
||||
TextSection(
|
||||
heading="Payment Terms",
|
||||
body="Payment is due within 30 days.\n\nPlease reference the invoice number.",
|
||||
),
|
||||
SignatureSection(
|
||||
heading="Authorised By",
|
||||
signatories=["Jane Smith, CEO", "Bob Jones, CFO"],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _simple_meta() -> DocumentMeta:
|
||||
return DocumentMeta(
|
||||
title="Invoice",
|
||||
@@ -170,6 +199,82 @@ def test_generated_document_optional_fields() -> None:
|
||||
assert doc.reference_number is None
|
||||
|
||||
|
||||
# ── Jinja rendering ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_render_produces_html(agent: PdfCreateAgent) -> None:
|
||||
doc = _invoice_doc()
|
||||
html = agent._render(doc)
|
||||
assert "<!DOCTYPE html>" in html
|
||||
assert "Invoice" in html
|
||||
assert "INV-001" in html
|
||||
|
||||
|
||||
def test_render_includes_all_section_types(agent: PdfCreateAgent) -> None:
|
||||
doc = GeneratedDocument(
|
||||
title="All Sections",
|
||||
sections=[
|
||||
TextSection(body="Some prose text."),
|
||||
KeyValueSection(pairs=[("Key", "Value")]),
|
||||
LineItemsSection(columns=["A", "B"], rows=[["1", "2"]]),
|
||||
BulletListSection(items=["item one"]),
|
||||
SignatureSection(signatories=["Alice"]),
|
||||
],
|
||||
)
|
||||
html = agent._render(doc)
|
||||
assert "Some prose text." in html
|
||||
assert "Key" in html and "Value" in html
|
||||
assert "<th>" in html
|
||||
assert "item one" in html
|
||||
assert "Alice" in html
|
||||
|
||||
|
||||
def test_render_escapes_html_in_content(agent: PdfCreateAgent) -> None:
|
||||
doc = GeneratedDocument(
|
||||
title="XSS Test",
|
||||
sections=[TextSection(body="<script>alert('xss')</script>")],
|
||||
)
|
||||
html = agent._render(doc)
|
||||
assert "<script>" not in html
|
||||
assert "<script>" in html
|
||||
|
||||
|
||||
def test_render_total_row_present(agent: PdfCreateAgent) -> None:
|
||||
doc = GeneratedDocument(
|
||||
title="Table",
|
||||
sections=[
|
||||
LineItemsSection(
|
||||
columns=["Item", "Total"],
|
||||
rows=[["Widget", "$10"]],
|
||||
total_row=["Total", "$10"],
|
||||
)
|
||||
],
|
||||
)
|
||||
html = agent._render(doc)
|
||||
assert "total-row" in html
|
||||
|
||||
|
||||
def test_render_no_total_row_skips_tfoot(agent: PdfCreateAgent) -> None:
|
||||
doc = GeneratedDocument(
|
||||
title="Table",
|
||||
sections=[LineItemsSection(columns=["Item"], rows=[["Widget"]])],
|
||||
)
|
||||
html = agent._render(doc)
|
||||
assert "<tfoot>" not in html
|
||||
|
||||
|
||||
def test_render_subtitle_and_reference(agent: PdfCreateAgent) -> None:
|
||||
doc = GeneratedDocument(
|
||||
title="My Doc",
|
||||
subtitle="Subtitle Here",
|
||||
reference_number="REF-42",
|
||||
sections=[TextSection(body="Content.")],
|
||||
)
|
||||
html = agent._render(doc)
|
||||
assert "Subtitle Here" in html
|
||||
assert "REF-42" in html
|
||||
|
||||
|
||||
# ── _safe_filename ────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -291,9 +396,8 @@ async def test_orchestrate_returns_plan_step(agent: PdfCreateAgent) -> None:
|
||||
assert step.tool == AgentToolId.CREATE_PDF_FROM_HTML_AGENT
|
||||
assert isinstance(step.parameters, CreatePdfFromHtmlAgentParams)
|
||||
assert step.parameters.filename.endswith(".pdf")
|
||||
parsed = json.loads(step.parameters.document)
|
||||
assert parsed["title"] == "Invoice"
|
||||
assert parsed["sections"]
|
||||
assert "<!DOCTYPE html>" in step.parameters.html_content
|
||||
assert "Invoice" in step.parameters.html_content
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@@ -364,9 +468,9 @@ async def test_orchestrate_assembles_multiple_chunks(agent: PdfCreateAgent) -> N
|
||||
result = await agent.orchestrate(_orchestrator_request("Create a multi-chunk doc"))
|
||||
|
||||
assert isinstance(result, EditPlanResponse)
|
||||
document = result.steps[0].parameters.document # type: ignore[union-attr]
|
||||
assert "Introduction text." in document
|
||||
assert "Details" in document
|
||||
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
|
||||
assert "Introduction text." in html
|
||||
assert "Details" in html
|
||||
|
||||
|
||||
# ── Style inference ───────────────────────────────────────────────────────────────────────────────
|
||||
@@ -378,7 +482,7 @@ async def test_orchestrate_applies_planner_inferred_style(agent: PdfCreateAgent)
|
||||
meta = DocumentMeta(
|
||||
title="Styled Doc",
|
||||
tone_brief="Professional.",
|
||||
style_primary_color="#ff00ff",
|
||||
style_primary_color="magenta",
|
||||
)
|
||||
sections = _simple_sections()
|
||||
written = _written_sections()
|
||||
@@ -395,35 +499,50 @@ async def test_orchestrate_applies_planner_inferred_style(agent: PdfCreateAgent)
|
||||
result = await agent.orchestrate(_orchestrator_request("Make an invoice, magenta styling"))
|
||||
|
||||
assert isinstance(result, EditPlanResponse)
|
||||
document = result.steps[0].parameters.document # type: ignore[union-attr]
|
||||
assert json.loads(document)["style"]["primaryColor"] == "#ff00ff"
|
||||
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
|
||||
assert "magenta" in html
|
||||
|
||||
|
||||
def test_document_style_keeps_only_six_digit_hex() -> None:
|
||||
"""Only #RRGGBB hex is kept; named colours and other formats drop to None."""
|
||||
safe = DocumentStyle(primary_color="#1e3a5f", background_color="#ffffff", body_text_color="#1A1A1A")
|
||||
def test_render_applies_style(agent: PdfCreateAgent) -> None:
|
||||
"""DocumentStyle fields are injected as CSS custom properties in the rendered HTML."""
|
||||
doc = GeneratedDocument(
|
||||
title="Styled",
|
||||
sections=[TextSection(body="Content.")],
|
||||
style=DocumentStyle(primary_color="magenta", background_color="#111111"),
|
||||
)
|
||||
html = agent._render(doc)
|
||||
assert "--color-primary: magenta" in html
|
||||
assert "--color-bg: #111111" in html
|
||||
|
||||
|
||||
def test_document_style_drops_unsafe_colors() -> None:
|
||||
"""Unsafe colours (not named/hex) are dropped, closing the <style> url() injection."""
|
||||
safe = DocumentStyle(primary_color="navy", background_color="#1e3a5f", body_text_color="#fff")
|
||||
assert (safe.primary_color, safe.background_color, safe.body_text_color) == (
|
||||
"navy",
|
||||
"#1e3a5f",
|
||||
"#ffffff",
|
||||
"#1A1A1A",
|
||||
"#fff",
|
||||
)
|
||||
|
||||
assert DocumentStyle(primary_color="navy").primary_color is None
|
||||
assert DocumentStyle(primary_color="#fff").primary_color is None
|
||||
assert DocumentStyle(primary_color="#1e3a5f00").primary_color is None
|
||||
assert DocumentStyle(primary_color="rgb(255, 0, 0)").primary_color is None
|
||||
assert DocumentStyle(background_color="teal darken-2").background_color is None
|
||||
unsafe = DocumentStyle(
|
||||
primary_color="red; background: url(http://evil.test/steal)",
|
||||
background_color="expression(alert(1))",
|
||||
body_text_color="navy; }",
|
||||
)
|
||||
assert unsafe.primary_color is None
|
||||
assert unsafe.background_color is None
|
||||
assert unsafe.body_text_color is None
|
||||
# A trailing newline must not slip a value through (fullmatch, not $-before-newline).
|
||||
assert DocumentStyle(primary_color="#1e3a5f\n").primary_color is None
|
||||
assert DocumentStyle(primary_color="navy\n").primary_color is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_orchestrate_drops_non_hex_planner_colour(agent: PdfCreateAgent) -> None:
|
||||
"""A non-hex colour inferred by the meta planner is dropped before the document is emitted."""
|
||||
async def test_orchestrate_drops_unsafe_planner_color(agent: PdfCreateAgent) -> None:
|
||||
"""An unsafe colour inferred by the meta planner never reaches the rendered HTML."""
|
||||
meta = DocumentMeta(
|
||||
title="Doc",
|
||||
tone_brief="Professional.",
|
||||
style_primary_color="rgb(0, 0, 255)",
|
||||
style_primary_color="blue; background: url(http://evil.test/)",
|
||||
)
|
||||
sections = _simple_sections()
|
||||
written = _written_sections()
|
||||
@@ -440,6 +559,6 @@ async def test_orchestrate_drops_non_hex_planner_colour(agent: PdfCreateAgent) -
|
||||
result = await agent.orchestrate(_orchestrator_request("make it blue"))
|
||||
|
||||
assert isinstance(result, EditPlanResponse)
|
||||
document = result.steps[0].parameters.document # type: ignore[union-attr]
|
||||
assert "rgb(" not in document
|
||||
assert json.loads(document)["style"]["primaryColor"] is None
|
||||
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
|
||||
assert "evil.test" not in html
|
||||
assert "url(" not in html
|
||||
|
||||
Generated
+2
@@ -604,6 +604,7 @@ version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "jinja2" },
|
||||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "pgvector" },
|
||||
{ name = "posthog" },
|
||||
@@ -630,6 +631,7 @@ dev = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.116.0" },
|
||||
{ name = "jinja2", specifier = ">=3.1.0" },
|
||||
{ name = "opentelemetry-sdk", specifier = ">=1.39.0" },
|
||||
{ name = "pgvector", specifier = ">=0.3.6" },
|
||||
{ name = "posthog", specifier = ">=3.0.0" },
|
||||
|
||||
@@ -563,6 +563,39 @@ close = "Close"
|
||||
error = "Error"
|
||||
expand = "Expand"
|
||||
|
||||
[admin.formDetection]
|
||||
active = "Active"
|
||||
description = "Install the AI model used to auto-detect form fields. The selected model is downloaded on demand (about 40-100MB) into the configs volume and is not bundled with Stirling-PDF."
|
||||
enableFeature = "Enable feature"
|
||||
engineDescription = "Browser keeps the PDF on the device (downloads a ~12MB runtime once, then cached); Server runs it on the backend; Auto prefers the browser and falls back to the server."
|
||||
engineLabel = "Where detection runs"
|
||||
install = "Install"
|
||||
license = "License"
|
||||
notAvailable = "This catalog entry has no download URL/checksum configured yet, so it cannot be installed."
|
||||
notWritable = "The model directory is not writable; check the configs volume mount."
|
||||
selectModel = "Model"
|
||||
selectPlaceholder = "Select a model"
|
||||
status = "Status"
|
||||
switch = "Switch to this model"
|
||||
title = "AI Form Detection"
|
||||
uninstall = "Uninstall"
|
||||
|
||||
[admin.formDetection.airgap]
|
||||
hide = "Hide offline install instructions"
|
||||
intro = "No internet on the server? Install the model manually:"
|
||||
noSha = "(checksum not set)"
|
||||
show = "Air-gapped / offline install instructions"
|
||||
step1 = "1. On a machine with internet, download the model file:"
|
||||
step2 = "2. Verify its SHA-256 checksum matches:"
|
||||
step3 = "3. Copy it onto the Stirling-PDF server into the model directory:"
|
||||
step4 = "4. Set formDetection.activeModelId to this model's id in settings.yml, then restart (an installed model is auto-detected on boot). <configs> is the configs volume (e.g. /configs in Docker). The Docker image also pre-bundles this model, so it is ready with no download."
|
||||
|
||||
[admin.formDetection.engine]
|
||||
auto = "Auto"
|
||||
browser = "Browser"
|
||||
server = "Server"
|
||||
serverUnavailable = "The server engine is not bundled in this build, so detection runs in the browser. Use Auto or Browser."
|
||||
|
||||
[admin.settings]
|
||||
discard = "Discard"
|
||||
error = "Failed to save settings"
|
||||
@@ -1883,6 +1916,16 @@ bullet2 = "Creates a clean, valid filename from the detected title"
|
||||
bullet3 = "Keeps the original name if no suitable title is found"
|
||||
title = "Smart Renaming"
|
||||
|
||||
[autoFormDetection]
|
||||
loading = "Detecting form fields..."
|
||||
submit = "Detect & make fillable"
|
||||
|
||||
[autoFormDetection.error]
|
||||
failed = "An error occurred while detecting form fields."
|
||||
|
||||
[autoFormDetection.results]
|
||||
title = "Detected Form Fields"
|
||||
|
||||
[automate]
|
||||
copyToSaved = "Copy to Saved"
|
||||
desc = "Build multi-step workflows by chaining together PDF actions. Ideal for recurring tasks."
|
||||
@@ -3608,8 +3651,6 @@ lastSynced = "Last synced"
|
||||
leaveShare = "Remove from my list"
|
||||
leaveShareFailed = "Could not remove the shared file."
|
||||
leaveShareSuccess = "Removed from your shared list."
|
||||
loadFailed = "Couldn't load your recent files"
|
||||
loadFailedRetry = "Try again"
|
||||
loadingFiles = "Loading files..."
|
||||
localOnly = "Local only"
|
||||
makeCopy = "Make a copy"
|
||||
@@ -3736,9 +3777,20 @@ uploadToServer = "Upload to server"
|
||||
versionHistory = "Version history"
|
||||
|
||||
[fileSidebar.groupsModal]
|
||||
add = "Add"
|
||||
addLabel = "Add a label…"
|
||||
categoryIconAria = "Choose an icon for {{name}}"
|
||||
createCategory = "Create category"
|
||||
delete = "Delete category"
|
||||
done = "Done"
|
||||
reset = "Show all"
|
||||
subtitle = "Show or hide categories in the files sidebar."
|
||||
hide = "Hide category"
|
||||
newCategory = "New category name…"
|
||||
removeLabel = "Remove {{name}}"
|
||||
rename = "Rename"
|
||||
reset = "Reset to defaults"
|
||||
search = "Search labels…"
|
||||
show = "Show category"
|
||||
subtitle = "Group your files into parent categories. Add existing or new labels to a category, rename it, or create your own. Files in none of your categories appear under “Other”."
|
||||
title = "Sidebar categories"
|
||||
|
||||
[filesPage]
|
||||
@@ -4308,6 +4360,11 @@ desc = "Highlight, draw, add notes and shapes in the viewer"
|
||||
tags = "annotate,highlight,draw,markup,comment,notes,review,redline,feedback,markup tools,sticky notes,shapes,arrows,text box,freehand"
|
||||
title = "Annotate"
|
||||
|
||||
[home.autoFormDetection]
|
||||
desc = "Automatically detect form fields with AI and make your PDF fillable."
|
||||
tags = "form,detect,fillable,acroform,auto detect,ai,fields,make fillable,form detection"
|
||||
title = "Auto Form Detection"
|
||||
|
||||
[home.automate]
|
||||
desc = "Build multi-step workflows by chaining together PDF actions. Ideal for recurring tasks."
|
||||
tags = "workflow,sequence,automation,automate,batch,batch processing,pipeline,chain,multi-step,recurring,scheduled,automatic,process multiple,bulk operations"
|
||||
@@ -6005,11 +6062,31 @@ summaryMore = "{{first}}, {{second}} and {{more}} more"
|
||||
summaryTwo = "{{first}} and {{second}}"
|
||||
|
||||
[policies.labels]
|
||||
categoryCount = "categories"
|
||||
hideCategory = "Hide category"
|
||||
labelCount = "labels"
|
||||
sharedNote = "These labels are built in and shared across your whole team."
|
||||
showCategory = "Show category"
|
||||
add = "Add"
|
||||
addPlaceholder = "Add a label…"
|
||||
customNote = "Customized for your team."
|
||||
defaultNote = "Using the built-in default, shared with your team."
|
||||
duplicate = "\"{{name}}\" already exists."
|
||||
edit = "Edit labels"
|
||||
empty = "No labels yet."
|
||||
export = "Export JSON"
|
||||
iconAria = "Choose an icon for {{name}}"
|
||||
import = "Import JSON"
|
||||
importError = "Couldn't import that file."
|
||||
managedNote = "Team labels are managed by your team leader."
|
||||
modalSubtitle = "Shared with your whole team. The classifier picks the labels that fit each document."
|
||||
modalTitle = "Classification labels"
|
||||
pickIcon = "Use {{label}} icon"
|
||||
removeAria = "Remove {{name}}"
|
||||
resetToDefault = "Reset to default"
|
||||
saveForTeam = "Save for team"
|
||||
saving = "Saving…"
|
||||
sectionLabel = "Classification labels"
|
||||
startFromScratch = "Start from scratch"
|
||||
teamCount = "team labels"
|
||||
tooLong = "Labels can be at most {{max}} characters."
|
||||
ungrouped = "Ungrouped"
|
||||
view = "View labels"
|
||||
|
||||
[policies.pii]
|
||||
account = "Account numbers (labelled)"
|
||||
@@ -6211,10 +6288,6 @@ muted = "muted"
|
||||
nameLabel = "Name"
|
||||
namePlaceholder = "e.g. Compliance escalation"
|
||||
|
||||
[portal.agentBuilder.status]
|
||||
draft = "Draft"
|
||||
published = "Published"
|
||||
|
||||
[portal.agentBuilder.tabs]
|
||||
evals = "Evals"
|
||||
scenarios = "Scenarios"
|
||||
@@ -6438,7 +6511,7 @@ install = "Install"
|
||||
usage = "Usage"
|
||||
|
||||
[portal.catalogue.detail.locked]
|
||||
description = "{{name}} is included from the {{plan}}. Upgrade to embed it."
|
||||
description = "{{name}} is included from the {{tier}} plan. Upgrade to embed it."
|
||||
title = "Not available on your plan"
|
||||
|
||||
[portal.catalogue.detail.preview]
|
||||
@@ -6505,35 +6578,6 @@ title = "No components available"
|
||||
description = "GA components are available on Pay-as-you-go; a few Beta components are enterprise-only. Locked cards show an upgrade nudge."
|
||||
title = "Some components need a paid plan"
|
||||
|
||||
[portal.connections]
|
||||
createTitle = "New S3 connection"
|
||||
delete = "Delete"
|
||||
edit = "Edit"
|
||||
editTitle = "Edit S3 connection"
|
||||
subtitle = "Reusable S3 credentials that sources and pipeline outputs connect to."
|
||||
|
||||
[portal.connections.actions]
|
||||
new = "New connection"
|
||||
|
||||
[portal.connections.empty]
|
||||
description = "Add an S3 connection to reuse the same bucket and credentials across sources and pipeline outputs."
|
||||
title = "No connections yet"
|
||||
|
||||
[portal.connections.picker]
|
||||
cancel = "Cancel"
|
||||
createNew = "New connection..."
|
||||
placeholder = "Select a connection"
|
||||
save = "Save connection"
|
||||
|
||||
[portal.connections.s3.fields]
|
||||
name = "Connection name"
|
||||
namePlaceholder = "e.g. Claims bucket"
|
||||
|
||||
[portal.connections.table]
|
||||
bucket = "Bucket"
|
||||
name = "Name"
|
||||
region = "Region"
|
||||
|
||||
[portal.docs.authentication]
|
||||
codeCaption = "every request"
|
||||
eyebrow = "GETTING STARTED"
|
||||
@@ -7345,6 +7389,10 @@ operations_one = "Operation ({{count}})"
|
||||
operations_other = "Operations ({{count}})"
|
||||
output = "Output"
|
||||
removeStep = "Remove operation"
|
||||
s3Configure = "Configure"
|
||||
s3Done = "Done"
|
||||
s3ModalTitle = "Amazon S3 output"
|
||||
s3NotConfigured = "Not configured"
|
||||
s3PrefixHelp = "Outputs are uploaded under this key prefix."
|
||||
save = "Save changes"
|
||||
scheduleEvery = "Run every"
|
||||
@@ -7415,18 +7463,12 @@ manual = "Manual"
|
||||
schedule = "Scheduled"
|
||||
|
||||
[portal.policies]
|
||||
defaultName = "{{category}} Policy"
|
||||
subtitle = "Standing automations that enforce a tool pipeline on every document. Each policy fires on upload or export, runs its tool chain, and saves the enforced version alongside the original."
|
||||
title = "Policies"
|
||||
|
||||
[portal.policies.card]
|
||||
comingSoon = "Upgrade to Enterprise"
|
||||
notSetUp = "Not set up"
|
||||
requiresAiEngine = "Requires AI engine"
|
||||
|
||||
[portal.policies.categories.classification]
|
||||
desc = "Identify each document's type on upload and tag its metadata for filing and search."
|
||||
label = "Classification"
|
||||
|
||||
[portal.policies.categories.compliance]
|
||||
desc = "Enforce HIPAA, GDPR, SOC 2, or FedRAMP requirements on every document."
|
||||
@@ -7451,13 +7493,6 @@ label = "Security"
|
||||
[portal.policies.config]
|
||||
scopeAll = "All documents"
|
||||
|
||||
[portal.policies.config.classification]
|
||||
summary = "Classifies every uploaded document against your team's labels and tags its metadata."
|
||||
|
||||
[portal.policies.config.classification.rules]
|
||||
0 = "Classify"
|
||||
1 = "Tag metadata"
|
||||
|
||||
[portal.policies.config.compliance]
|
||||
summary = "Validates documents against regulatory frameworks before they leave the system."
|
||||
|
||||
@@ -7552,7 +7587,6 @@ title = "No activity yet"
|
||||
[portal.policies.endpoints]
|
||||
addWatermark = "Watermark"
|
||||
autoRedact = "Redact PII"
|
||||
classifyAndLabel = "Classify"
|
||||
compressPdf = "Compress"
|
||||
flatten = "Flatten"
|
||||
ocrPdf = "OCR"
|
||||
@@ -7595,10 +7629,6 @@ continue = "Continue"
|
||||
enablePolicy = "Enable policy"
|
||||
saveChanges = "Save changes"
|
||||
|
||||
[portal.policies.wizard.capability.classify]
|
||||
desc = "Identifies the document's type from your team's labels and tags it, so it files and searches by category."
|
||||
label = "Classify the document"
|
||||
|
||||
[portal.policies.wizard.capability.compress]
|
||||
desc = "Compresses the document to a smaller file size."
|
||||
label = "Reduce file size"
|
||||
@@ -7623,10 +7653,6 @@ label = "Strip active content"
|
||||
desc = "Stamps a visible mark (e.g. “Confidential”) across every page."
|
||||
label = "Apply a watermark"
|
||||
|
||||
[portal.policies.wizard.classification]
|
||||
description = "Every uploaded document is classified against the built-in labels and tagged with the types that fit. The label set is shared across your whole team."
|
||||
labelsHeading = "Classification labels"
|
||||
|
||||
[portal.policies.wizard.errors]
|
||||
noTools = "Enable at least one tool in the workflow first."
|
||||
saveFailed = "Couldn't save the policy. Please try again."
|
||||
@@ -8008,29 +8034,34 @@ primaryNav = "Primary navigation"
|
||||
switchApp = "Switch app"
|
||||
|
||||
[portal.sources]
|
||||
subtitle = "Reusable input connections that feed documents into Stirling. Configure a source once, then reference it from any number of pipelines."
|
||||
subtitle = "Reusable input connections that feed documents into Stirling. Configure a connection once, then reference it from any number of policies. Click a row for its config and which policies use it."
|
||||
title = "Sources"
|
||||
|
||||
[portal.sources.actions]
|
||||
agentBuilder = "Agent Builder"
|
||||
connectSource = "Connect source"
|
||||
|
||||
[portal.sources.builder]
|
||||
back = "Back to sources"
|
||||
cancel = "Cancel"
|
||||
create = "Create source"
|
||||
createTitle = "Connect a source"
|
||||
delete = "Delete"
|
||||
editTitle = "Edit source"
|
||||
enabled = "Enabled"
|
||||
save = "Save changes"
|
||||
|
||||
[portal.sources.delete]
|
||||
body = "Delete \"{{name}}\"? This can't be undone. Policies that reference it would need to be updated."
|
||||
cancel = "Cancel"
|
||||
confirm = "Delete"
|
||||
title = "Delete source?"
|
||||
|
||||
[portal.sources.detail]
|
||||
closeAriaLabel = "Close detail"
|
||||
delete = "Delete source"
|
||||
docs24h = "Last 24h"
|
||||
docs30d = "Last 30 days"
|
||||
docsTotal = "Total seen"
|
||||
docsTrend = "Documents over the last 30 days"
|
||||
documents = "Documents"
|
||||
edit = "Edit"
|
||||
notReferenced = "Not referenced by any policy, so it's safe to delete."
|
||||
pause = "Pause"
|
||||
resume = "Resume"
|
||||
subtitle = "{{type}} · {{status}}"
|
||||
usedBy = "Used by"
|
||||
|
||||
[portal.sources.empty]
|
||||
description = "Connect a folder (and, soon, cloud storage) so your policies have somewhere to pull documents from."
|
||||
title = "No sources connected yet"
|
||||
@@ -8046,15 +8077,10 @@ disabled = "Disabled"
|
||||
unused = "Unused"
|
||||
|
||||
[portal.sources.table]
|
||||
documents = "Documents"
|
||||
source = "Source"
|
||||
status = "Status"
|
||||
usedBy = "Policies"
|
||||
|
||||
[portal.sources.tabs]
|
||||
connections = "Connections"
|
||||
sources = "Sources"
|
||||
|
||||
[portal.sources.types.editor]
|
||||
description = "Documents your team has processed in the editor, across policy and AI runs."
|
||||
label = "Editor"
|
||||
@@ -8102,10 +8128,6 @@ label = "Access key ID"
|
||||
label = "Bucket"
|
||||
placeholder = "my-company-inbox"
|
||||
|
||||
[portal.sources.types.s3.fields.connection]
|
||||
helperText = "The stored connection holding the bucket and credentials. Reused by every source and pipeline output that references it."
|
||||
label = "Connection"
|
||||
|
||||
[portal.sources.types.s3.fields.endpoint]
|
||||
helperText = "Leave blank for Amazon S3. Set to use an S3-compatible service such as MinIO."
|
||||
label = "Custom endpoint"
|
||||
@@ -8135,14 +8157,21 @@ label = "Secret access key"
|
||||
label = "Source"
|
||||
|
||||
[portal.sources.wizard]
|
||||
back = "Back"
|
||||
cancel = "Cancel"
|
||||
continue = "Continue"
|
||||
editTitle = "Edit source"
|
||||
name = "Name"
|
||||
namePlaceholder = "e.g. Claims intake"
|
||||
save = "Save changes"
|
||||
subtitle = "Step {{current}} of {{total}} · {{label}}"
|
||||
title = "Connect a source"
|
||||
type = "Type"
|
||||
|
||||
[portal.tier]
|
||||
enterprise = "Enterprise plan"
|
||||
free = "Editor plan"
|
||||
pro = "Processor plan"
|
||||
[portal.sources.wizard.steps]
|
||||
chooseType = "Choose type"
|
||||
configure = "Configure"
|
||||
review = "Review & connect"
|
||||
|
||||
[portal.usage]
|
||||
managePayment = "Manage Payment"
|
||||
@@ -10158,19 +10187,6 @@ resetPw = "Reset password"
|
||||
suspend = "Suspend"
|
||||
unlock = "Unlock account"
|
||||
|
||||
[users.activity]
|
||||
daysAgo = "{{count}}d ago"
|
||||
hoursAgo = "{{count}}h ago"
|
||||
justNow = "Just now"
|
||||
minutesAgo = "{{count}}m ago"
|
||||
monthsAgo_one = "{{count}} month ago"
|
||||
monthsAgo_other = "{{count}} months ago"
|
||||
never = "Never"
|
||||
weeksAgo_one = "{{count}} week ago"
|
||||
weeksAgo_other = "{{count}} weeks ago"
|
||||
yearsAgo_one = "{{count}} year ago"
|
||||
yearsAgo_other = "{{count}} years ago"
|
||||
|
||||
[users.cap]
|
||||
addProcessor = "+ Processor"
|
||||
approver = "Approves policy"
|
||||
@@ -10241,9 +10257,7 @@ by = "Invited by {{who}}"
|
||||
cancel = "Cancel"
|
||||
count = "{{count}} pending"
|
||||
desc = "Invited people who haven't joined yet. They hold a seat until they accept."
|
||||
expiresInDays_one = "Expires in {{count}} day"
|
||||
expiresInDays_other = "Expires in {{count}} days"
|
||||
expiresToday = "Expires today"
|
||||
expires = "Expires"
|
||||
title = "Pending invitations"
|
||||
|
||||
[users.loadError]
|
||||
|
||||
@@ -265,6 +265,11 @@
|
||||
"title": "Fill Form - Stirling PDF",
|
||||
"description": "Fill PDF form fields interactively with a visual editor"
|
||||
},
|
||||
"autoFormDetection": {
|
||||
"image": "/og_images/home.png",
|
||||
"title": "Auto Form Detection - Stirling PDF",
|
||||
"description": "Automatically detect form fields with AI and make your PDF fillable."
|
||||
},
|
||||
"multiTool": {
|
||||
"image": "/og_images/multi-tool.png",
|
||||
"title": "Multi-Tool - Stirling PDF",
|
||||
@@ -559,6 +564,7 @@
|
||||
"/booklet-imposition": "bookletImposition",
|
||||
"/pdf-text-editor": "pdfTextEditor",
|
||||
"/form-fill": "formFill",
|
||||
"/auto-form-detection": "autoFormDetection",
|
||||
"/multi-tool": "multiTool",
|
||||
"/read": "read",
|
||||
"/automate": "automate",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
|
||||
"productName": "Stirling PDF",
|
||||
"mainBinaryName": "Stirling-PDF",
|
||||
"version": "2.14.2",
|
||||
"version": "2.14.1",
|
||||
"identifier": "stirling.pdf.dev",
|
||||
"build": {
|
||||
"frontendDist": "../dist",
|
||||
|
||||
@@ -42,8 +42,7 @@ const FileManager: React.FC<FileManagerProps> = ({ selectedTool }) => {
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [isMobile, setIsMobile] = useState(false);
|
||||
|
||||
const { loadRecentFiles, handleRemoveFile, loading, loadError } =
|
||||
useFileManager();
|
||||
const { loadRecentFiles, handleRemoveFile, loading } = useFileManager();
|
||||
const { actions: fileActions } = useFileActions();
|
||||
|
||||
// Get active file IDs from FileContext to show which files are already loaded
|
||||
@@ -241,7 +240,6 @@ const FileManager: React.FC<FileManagerProps> = ({ selectedTool }) => {
|
||||
modalHeight={modalHeight}
|
||||
refreshRecentFiles={refreshRecentFiles}
|
||||
isLoading={loading}
|
||||
loadError={loadError}
|
||||
activeFileIds={activeFileIds}
|
||||
>
|
||||
{isMobile ? <MobileLayout /> : <DesktopLayout />}
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
import { render, screen, fireEvent } from "@testing-library/react";
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
|
||||
// Mock i18n so assertions can rely on the English default strings.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({
|
||||
t: (_key: string, fallback?: string) => fallback ?? _key,
|
||||
}),
|
||||
}));
|
||||
|
||||
// Isolate FileListArea's branching logic from its heavy children — we only
|
||||
// care which state it renders, not how each child looks.
|
||||
vi.mock("@app/components/fileManager/EmptyFilesState", () => ({
|
||||
default: () => <div>No recent files</div>,
|
||||
}));
|
||||
vi.mock("@app/components/fileManager/FileListItem", () => ({
|
||||
default: ({ file }: { file: { name: string } }) => <div>{file.name}</div>,
|
||||
}));
|
||||
vi.mock("@app/components/fileManager/FileHistoryGroup", () => ({
|
||||
default: () => null,
|
||||
}));
|
||||
vi.mock("@app/ui/Button", () => ({
|
||||
Button: ({
|
||||
children,
|
||||
onClick,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
onClick?: () => void;
|
||||
}) => <button onClick={onClick}>{children}</button>,
|
||||
}));
|
||||
|
||||
const contextValue = vi.hoisted(() => ({
|
||||
current: {} as Record<string, unknown>,
|
||||
}));
|
||||
vi.mock("@app/contexts/FileManagerContext", () => ({
|
||||
useFileManagerContext: () => contextValue.current,
|
||||
}));
|
||||
|
||||
import FileListArea from "@app/components/fileManager/FileListArea";
|
||||
|
||||
function baseContext(overrides: Record<string, unknown>) {
|
||||
return {
|
||||
activeSource: "recent",
|
||||
recentFiles: [],
|
||||
filteredFiles: [],
|
||||
selectedFilesSet: new Set(),
|
||||
expandedFileIds: new Set(),
|
||||
loadedHistoryFiles: new Map(),
|
||||
onFileSelect: vi.fn(),
|
||||
onFileRemove: vi.fn(),
|
||||
onHistoryFileRemove: vi.fn(),
|
||||
onFileDoubleClick: vi.fn(),
|
||||
onDownloadSingle: vi.fn(),
|
||||
isLoading: false,
|
||||
loadError: false,
|
||||
activeFileIds: [],
|
||||
refreshRecentFiles: vi.fn(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("FileListArea recent-files states", () => {
|
||||
beforeEach(() => {
|
||||
contextValue.current = baseContext({});
|
||||
});
|
||||
|
||||
it("shows the empty state when there are no files and no error", () => {
|
||||
contextValue.current = baseContext({ isLoading: false, loadError: false });
|
||||
render(
|
||||
<MantineProvider>
|
||||
<FileListArea scrollAreaHeight="200px" />
|
||||
</MantineProvider>,
|
||||
);
|
||||
expect(screen.getByText("No recent files")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Try again")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the loading state while loading with no files yet", () => {
|
||||
contextValue.current = baseContext({ isLoading: true });
|
||||
render(
|
||||
<MantineProvider>
|
||||
<FileListArea scrollAreaHeight="200px" />
|
||||
</MantineProvider>,
|
||||
);
|
||||
expect(screen.getByText("Loading files...")).toBeInTheDocument();
|
||||
expect(screen.queryByText("No recent files")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows an error state with a retry action when the load fails (not the empty state)", () => {
|
||||
const refreshRecentFiles = vi.fn();
|
||||
contextValue.current = baseContext({
|
||||
isLoading: false,
|
||||
loadError: true,
|
||||
refreshRecentFiles,
|
||||
});
|
||||
render(
|
||||
<MantineProvider>
|
||||
<FileListArea scrollAreaHeight="200px" />
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
// The failed load must be visually distinct from a genuinely empty history.
|
||||
expect(
|
||||
screen.getByText("Couldn't load your recent files"),
|
||||
).toBeInTheDocument();
|
||||
expect(screen.queryByText("No recent files")).not.toBeInTheDocument();
|
||||
|
||||
const retry = screen.getByText("Try again");
|
||||
fireEvent.click(retry);
|
||||
expect(refreshRecentFiles).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("renders the file list when files are present even if a partial load error occurred", () => {
|
||||
contextValue.current = baseContext({
|
||||
recentFiles: [{ id: "a" }],
|
||||
filteredFiles: [{ id: "a", name: "doc.pdf" }],
|
||||
loadError: true,
|
||||
});
|
||||
render(
|
||||
<MantineProvider>
|
||||
<FileListArea scrollAreaHeight="200px" />
|
||||
</MantineProvider>,
|
||||
);
|
||||
expect(screen.getByText("doc.pdf")).toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByText("Couldn't load your recent files"),
|
||||
).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("No recent files")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,10 @@
|
||||
import React from "react";
|
||||
import { Center, ScrollArea, Text, Stack } from "@mantine/core";
|
||||
import CloudIcon from "@mui/icons-material/Cloud";
|
||||
import ErrorOutlineIcon from "@mui/icons-material/ErrorOutlineRounded";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import FileListItem from "@app/components/fileManager/FileListItem";
|
||||
import FileHistoryGroup from "@app/components/fileManager/FileHistoryGroup";
|
||||
import EmptyFilesState from "@app/components/fileManager/EmptyFilesState";
|
||||
import { Button } from "@app/ui/Button";
|
||||
import { useFileManagerContext } from "@app/contexts/FileManagerContext";
|
||||
|
||||
interface FileListAreaProps {
|
||||
@@ -31,9 +29,7 @@ const FileListArea: React.FC<FileListAreaProps> = ({
|
||||
onFileDoubleClick,
|
||||
onDownloadSingle,
|
||||
isLoading,
|
||||
loadError,
|
||||
activeFileIds,
|
||||
refreshRecentFiles,
|
||||
} = useFileManagerContext();
|
||||
const { t } = useTranslation();
|
||||
|
||||
@@ -48,7 +44,15 @@ const FileListArea: React.FC<FileListAreaProps> = ({
|
||||
scrollbarSize={8}
|
||||
>
|
||||
<Stack gap={0}>
|
||||
{recentFiles.length > 0 ? (
|
||||
{recentFiles.length === 0 && !isLoading ? (
|
||||
<EmptyFilesState />
|
||||
) : recentFiles.length === 0 && isLoading ? (
|
||||
<Center style={{ height: "12.5rem" }}>
|
||||
<Text c="dimmed" ta="center">
|
||||
{t("fileManager.loadingFiles", "Loading files...")}
|
||||
</Text>
|
||||
</Center>
|
||||
) : (
|
||||
filteredFiles.map((file, index) => {
|
||||
// All files in filteredFiles are now leaf files only
|
||||
const historyFiles = loadedHistoryFiles.get(file.id) || [];
|
||||
@@ -80,39 +84,6 @@ const FileListArea: React.FC<FileListAreaProps> = ({
|
||||
</React.Fragment>
|
||||
);
|
||||
})
|
||||
) : isLoading ? (
|
||||
<Center style={{ height: "12.5rem" }}>
|
||||
<Text c="dimmed" ta="center">
|
||||
{t("fileManager.loadingFiles", "Loading files...")}
|
||||
</Text>
|
||||
</Center>
|
||||
) : loadError ? (
|
||||
<Center style={{ height: "12.5rem" }}>
|
||||
<Stack align="center" gap="sm">
|
||||
<ErrorOutlineIcon
|
||||
style={{
|
||||
fontSize: "3rem",
|
||||
color: "var(--mantine-color-gray-5)",
|
||||
}}
|
||||
/>
|
||||
<Text c="dimmed" ta="center">
|
||||
{t(
|
||||
"fileManager.loadFailed",
|
||||
"Couldn't load your recent files",
|
||||
)}
|
||||
</Text>
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
void refreshRecentFiles();
|
||||
}}
|
||||
>
|
||||
{t("fileManager.loadFailedRetry", "Try again")}
|
||||
</Button>
|
||||
</Stack>
|
||||
</Center>
|
||||
) : (
|
||||
<EmptyFilesState />
|
||||
)}
|
||||
</Stack>
|
||||
</ScrollArea>
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
// Shared source of truth for a policy category's outline icon, keyed by category
|
||||
// id (not a parallel icon-name vocabulary). Used by the editor's policy
|
||||
// definitions and the portal's catalogue cards, summaries, and setup wizard.
|
||||
|
||||
import type { ReactNode } from "react";
|
||||
import type { SxProps, Theme } from "@mui/material";
|
||||
import LayersOutlinedIcon from "@mui/icons-material/LayersOutlined";
|
||||
import ShieldOutlinedIcon from "@mui/icons-material/ShieldOutlined";
|
||||
import LabelOutlinedIcon from "@mui/icons-material/LabelOutlined";
|
||||
import CheckCircleOutlinedIcon from "@mui/icons-material/CheckCircleOutlined";
|
||||
import AltRouteOutlinedIcon from "@mui/icons-material/AltRouteOutlined";
|
||||
import ScheduleOutlinedIcon from "@mui/icons-material/ScheduleOutlined";
|
||||
|
||||
type MuiIcon = React.ComponentType<{ sx?: SxProps<Theme>; className?: string }>;
|
||||
|
||||
/** Policy category id → outline glyph. */
|
||||
const POLICY_CATEGORY_ICONS: Record<string, MuiIcon> = {
|
||||
ingestion: LayersOutlinedIcon,
|
||||
security: ShieldOutlinedIcon,
|
||||
classification: LabelOutlinedIcon,
|
||||
compliance: CheckCircleOutlinedIcon,
|
||||
routing: AltRouteOutlinedIcon,
|
||||
retention: ScheduleOutlinedIcon,
|
||||
};
|
||||
|
||||
const FALLBACK_ICON = LabelOutlinedIcon;
|
||||
|
||||
// Defaults to inheriting the surrounding font-size so a wrapping box controls size.
|
||||
export function policyCategoryIcon(
|
||||
categoryId: string,
|
||||
sx: SxProps<Theme> = { fontSize: "inherit" },
|
||||
className?: string,
|
||||
): ReactNode {
|
||||
const Icon = POLICY_CATEGORY_ICONS[categoryId] ?? FALLBACK_ICON;
|
||||
return <Icon sx={sx} className={className} />;
|
||||
}
|
||||
@@ -42,7 +42,6 @@ interface FileManagerContextValue {
|
||||
fileGroups: Map<FileId, StirlingFileStub[]>;
|
||||
loadedHistoryFiles: Map<FileId, StirlingFileStub[]>;
|
||||
isLoading: boolean;
|
||||
loadError: boolean;
|
||||
activeFileIds: FileId[];
|
||||
|
||||
// Handlers
|
||||
@@ -97,7 +96,6 @@ interface FileManagerProviderProps {
|
||||
modalHeight: string;
|
||||
refreshRecentFiles: () => Promise<void>;
|
||||
isLoading: boolean;
|
||||
loadError: boolean;
|
||||
activeFileIds: FileId[];
|
||||
maxSelectable?: number | null;
|
||||
}
|
||||
@@ -117,7 +115,6 @@ export const FileManagerProvider: React.FC<FileManagerProviderProps> = ({
|
||||
modalHeight,
|
||||
refreshRecentFiles,
|
||||
isLoading,
|
||||
loadError,
|
||||
activeFileIds,
|
||||
maxSelectable = null,
|
||||
}) => {
|
||||
@@ -1092,7 +1089,6 @@ export const FileManagerProvider: React.FC<FileManagerProviderProps> = ({
|
||||
fileGroups,
|
||||
loadedHistoryFiles,
|
||||
isLoading,
|
||||
loadError,
|
||||
activeFileIds,
|
||||
|
||||
// Handlers
|
||||
@@ -1135,7 +1131,6 @@ export const FileManagerProvider: React.FC<FileManagerProviderProps> = ({
|
||||
fileGroups,
|
||||
loadedHistoryFiles,
|
||||
isLoading,
|
||||
loadError,
|
||||
activeFileIds,
|
||||
handleSourceChange,
|
||||
handleStorageFilterChange,
|
||||
|
||||
@@ -30,6 +30,7 @@ import { addWatermarkOperationConfig } from "@app/hooks/tools/addWatermark/useAd
|
||||
import { addStampOperationConfig } from "@app/components/tools/addStamp/useAddStampOperation";
|
||||
import { addAttachmentsOperationConfig } from "@app/hooks/tools/addAttachments/useAddAttachmentsOperation";
|
||||
import { unlockPdfFormsOperationConfig } from "@app/hooks/tools/unlockPdfForms/useUnlockPdfFormsOperation";
|
||||
import { autoFormDetectionOperationConfig } from "@app/hooks/tools/autoFormDetection/useAutoFormDetectionOperation";
|
||||
import { singleLargePageOperationConfig } from "@app/hooks/tools/singleLargePage/useSingleLargePageOperation";
|
||||
import { ocrOperationConfig } from "@app/hooks/tools/ocr/useOCROperation";
|
||||
import { convertOperationConfig } from "@app/hooks/tools/convert/useConvertOperation";
|
||||
@@ -465,6 +466,39 @@ export function useTranslatedToolCatalog(): TranslatedToolCatalog {
|
||||
supportsAutomate: false,
|
||||
synonyms: ["form", "fill", "fillable", "input", "field", "acroform"],
|
||||
},
|
||||
autoFormDetection: {
|
||||
icon: (
|
||||
<LocalIcon
|
||||
icon="document-scanner-outline-rounded"
|
||||
width="1.5rem"
|
||||
height="1.5rem"
|
||||
/>
|
||||
),
|
||||
name: t("home.autoFormDetection.title", "Auto Form Detection"),
|
||||
component: lazy(
|
||||
() => import("@app/tools/autoFormDetection/AutoFormDetection"),
|
||||
),
|
||||
description: t(
|
||||
"home.autoFormDetection.desc",
|
||||
"Automatically detect form fields with AI and make your PDF fillable.",
|
||||
),
|
||||
categoryId: ToolCategoryId.STANDARD_TOOLS,
|
||||
subcategoryId: SubcategoryId.GENERAL,
|
||||
maxFiles: 1,
|
||||
endpoints: ["form-detection"],
|
||||
operationConfig: asRegistryConfig(autoFormDetectionOperationConfig),
|
||||
synonyms: [
|
||||
"form",
|
||||
"detect",
|
||||
"fillable",
|
||||
"acroform",
|
||||
"ai",
|
||||
"fields",
|
||||
"auto",
|
||||
],
|
||||
automationSettings: null,
|
||||
supportsAutomate: false,
|
||||
},
|
||||
changePermissions: {
|
||||
icon: <LocalIcon icon="lock-outline" width="1.5rem" height="1.5rem" />,
|
||||
name: t("home.changePermissions.title", "Change Permissions"),
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import {
|
||||
ToolType,
|
||||
useToolOperation,
|
||||
} from "@app/hooks/tools/shared/useToolOperation";
|
||||
import {
|
||||
FormDetectionModelStatus,
|
||||
FormDetectionCatalogEntry,
|
||||
} from "@app/hooks/useFormDetectionModelStatus";
|
||||
import apiClient from "@app/services/apiClient";
|
||||
import { createStandardErrorHandler } from "@app/utils/toolErrorHandler";
|
||||
import {
|
||||
AutoFormDetectionParameters,
|
||||
defaultParameters,
|
||||
} from "@app/hooks/tools/autoFormDetection/useAutoFormDetectionParameters";
|
||||
|
||||
const DETECT_ENDPOINT = "/api/v1/ai/form-detection/detect";
|
||||
const STATUS_URL = "/api/v1/ai/form-detection-model/status";
|
||||
|
||||
// Static function shared by the hook and the automation executor.
|
||||
export const buildAutoFormDetectionFormData = (
|
||||
parameters: AutoFormDetectionParameters,
|
||||
file: File,
|
||||
): FormData => {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
// Server path detects and applies the AcroForm in one call, returning the fillable PDF.
|
||||
formData.append("applyToPdf", "true");
|
||||
if (typeof parameters.confidence === "number") {
|
||||
formData.append("confThreshold", String(parameters.confidence));
|
||||
}
|
||||
return formData;
|
||||
};
|
||||
|
||||
function outputName(file: File): string {
|
||||
const base = (file.name || "document").replace(/\.pdf$/i, "");
|
||||
return `${base}_form.pdf`;
|
||||
}
|
||||
|
||||
async function serverDetect(
|
||||
parameters: AutoFormDetectionParameters,
|
||||
file: File,
|
||||
): Promise<File> {
|
||||
const res = await apiClient.post(
|
||||
DETECT_ENDPOINT,
|
||||
buildAutoFormDetectionFormData(parameters, file),
|
||||
{ responseType: "blob" },
|
||||
);
|
||||
return new File([res.data as Blob], outputName(file), {
|
||||
type: "application/pdf",
|
||||
});
|
||||
}
|
||||
|
||||
async function browserDetect(
|
||||
parameters: AutoFormDetectionParameters,
|
||||
file: File,
|
||||
entry: FormDetectionCatalogEntry,
|
||||
): Promise<File> {
|
||||
// Lazy-load the in-browser engine (onnxruntime-web + the ~12MB wasm) only when browser-mode
|
||||
// detection actually runs - it is never pulled into the initial bundle or loaded on the homepage.
|
||||
const { runBrowserDetection } =
|
||||
await import("@app/services/formDetection/runBrowserPipeline");
|
||||
const bytes = await file.arrayBuffer();
|
||||
const { appliedPdf } = await runBrowserDetection(
|
||||
bytes,
|
||||
entry,
|
||||
parameters.confidence,
|
||||
);
|
||||
return new File([new Uint8Array(appliedPdf)], outputName(file), {
|
||||
type: "application/pdf",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs detection where the admin configured it to run: 'server' (upload), 'browser' (in-browser
|
||||
* WASM only - the PDF never leaves the device, no fallback), or 'auto' (browser first, falling back
|
||||
* to the server on any browser-path error).
|
||||
*/
|
||||
async function processAutoFormDetection(
|
||||
parameters: AutoFormDetectionParameters,
|
||||
files: File[],
|
||||
): Promise<{ files: File[] }> {
|
||||
const file = files[0];
|
||||
|
||||
const status = (await apiClient.get(STATUS_URL))
|
||||
.data as FormDetectionModelStatus;
|
||||
const mode = status.executionMode ?? "auto";
|
||||
const activeEntry = (status.catalog ?? []).find(
|
||||
(c) => c.id === status.activeModelId,
|
||||
);
|
||||
|
||||
if (mode === "server" || !activeEntry) {
|
||||
return { files: [await serverDetect(parameters, file)] };
|
||||
}
|
||||
if (mode === "browser") {
|
||||
// Strict: never fall back to the server, so the PDF truly stays on the device.
|
||||
return { files: [await browserDetect(parameters, file, activeEntry)] };
|
||||
}
|
||||
// auto: prefer the browser, fall back to the server if it fails.
|
||||
try {
|
||||
return { files: [await browserDetect(parameters, file, activeEntry)] };
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[AutoFormDetection] in-browser engine failed; falling back to server",
|
||||
e,
|
||||
);
|
||||
return { files: [await serverDetect(parameters, file)] };
|
||||
}
|
||||
}
|
||||
|
||||
export const autoFormDetectionOperationConfig = {
|
||||
toolType: ToolType.custom,
|
||||
customProcessor: processAutoFormDetection,
|
||||
operationType: "autoFormDetection",
|
||||
// Used only for cloud/credit routing of the server path; execution is the custom processor.
|
||||
endpoint: DETECT_ENDPOINT,
|
||||
defaultParameters,
|
||||
} as const;
|
||||
|
||||
export const useAutoFormDetectionOperation = () => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
return useToolOperation<AutoFormDetectionParameters>({
|
||||
...autoFormDetectionOperationConfig,
|
||||
getErrorMessage: createStandardErrorHandler(
|
||||
t(
|
||||
"autoFormDetection.error.failed",
|
||||
"An error occurred while detecting form fields.",
|
||||
),
|
||||
),
|
||||
});
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user