From 2139edc157c69e40ab6ccc43bace35bbcee98b6e Mon Sep 17 00:00:00 2001 From: Reece Date: Mon, 23 Mar 2026 21:37:27 +0000 Subject: [PATCH] Backend smart foldering --- .../common/util/EmlProcessingUtils.java | 3 +- .../api/pipeline/PipelineProcessor.java | 27 ++++-- .../web/ReactRoutingController.java | 6 +- .../service/PdfJsonFallbackFontService.java | 7 +- .../software/SPDF/model/ApiEndpointTest.java | 3 +- .../service/LanguageServiceBasicTest.java | 4 +- .../SPDF/service/LanguageServiceTest.java | 12 +-- .../security/service/EmailService.java | 9 +- .../smartFolders/CardExpansionModal.tsx | 19 +++-- .../smartFolders/FilePreviewModal.tsx | 25 ++++-- .../smartFolders/SmartFolderWorkbenchView.tsx | 24 +++++- .../smartFolders/WatchFolderFileList.tsx | 38 +++++++-- .../src/core/hooks/useFolderAutomation.ts | 14 +--- frontend/src/core/types/smartFolders.ts | 1 + frontend/src/core/utils/automationExecutor.ts | 83 +++++++++++++++++++ 15 files changed, 213 insertions(+), 62 deletions(-) diff --git a/app/common/src/main/java/stirling/software/common/util/EmlProcessingUtils.java b/app/common/src/main/java/stirling/software/common/util/EmlProcessingUtils.java index 25af604c8c..0bb76f9921 100644 --- a/app/common/src/main/java/stirling/software/common/util/EmlProcessingUtils.java +++ b/app/common/src/main/java/stirling/software/common/util/EmlProcessingUtils.java @@ -341,7 +341,8 @@ public class EmlProcessingUtils { } private String getFallbackStyles() { - return """ + return + """ /* Minimal fallback - main CSS resource failed to load */ body { font-family: var(--font-family, Helvetica, sans-serif); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java index 1eac93b227..08dadbc48e 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java @@ -105,24 +105,37 @@ public class PipelineProcessor { boolean filtersApplied = false; for (PipelineOperation pipelineOperation : config.getOperations()) { String operation = pipelineOperation.getOperation(); - boolean isMultiInputOperation = apiDocService.isMultiInput(operation); + // Normalize to OpenAPI path format (leading "/") for apiDocService lookups. + // The frontend may omit the leading slash when building pipeline JSON. + String normalizedOperation = operation.startsWith("/") ? operation : "/" + operation; + boolean isMultiInputOperation = apiDocService.isMultiInput(normalizedOperation); log.info( "Running operation: {} isMultiInputOperation {}", - operation, + normalizedOperation, isMultiInputOperation); Map parameters = pipelineOperation.getParameters(); - List inputFileTypes = apiDocService.getExtensionTypes(false, operation); + List inputFileTypes = + apiDocService.getExtensionTypes(false, normalizedOperation); if (inputFileTypes == null) { inputFileTypes = new ArrayList<>(List.of("ALL")); } - if (!apiDocService.isValidOperation(operation, parameters)) { - log.error("Invalid operation or parameters: o:{} p:{}", operation, parameters); + if (!apiDocService.isValidOperation(normalizedOperation, parameters)) { + log.error( + "Invalid operation or parameters: o:{} p:{}", + normalizedOperation, + parameters); throw new IllegalArgumentException( - "Invalid operation: " + operation + " with parameters: " + parameters); + "Invalid operation: " + + normalizedOperation + + " with parameters: " + + parameters); } - String url = getBaseUrl() + operation; + // getBaseUrl() ends with "/"; strip leading "/" from normalizedOperation to avoid + // double slash + String operationPath = normalizedOperation.substring(1); + String url = getBaseUrl() + operationPath; List newOutputFiles = new ArrayList<>(); if (!isMultiInputOperation) { for (Resource file : outputFiles) { diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java b/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java index e903ed8e59..8f8dcb0b8e 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java @@ -178,7 +178,8 @@ public class ReactRoutingController { String escapedBaseUrlJs = JavaScriptUtils.javaScriptEscape(baseUrl); String serverUrl = "(window.location.origin + '" + escapedBaseUrlJs + "')"; - return """ + return + """ @@ -237,7 +238,8 @@ public class ReactRoutingController { String escapedBaseUrlJs = JavaScriptUtils.javaScriptEscape(baseUrl); String serverUrl = "(window.location.origin + '" + escapedBaseUrlJs + "')"; - return """ + return + """ diff --git a/app/core/src/main/java/stirling/software/SPDF/service/PdfJsonFallbackFontService.java b/app/core/src/main/java/stirling/software/SPDF/service/PdfJsonFallbackFontService.java index be51dd74eb..4f2a46f0c1 100644 --- a/app/core/src/main/java/stirling/software/SPDF/service/PdfJsonFallbackFontService.java +++ b/app/core/src/main/java/stirling/software/SPDF/service/PdfJsonFallbackFontService.java @@ -581,9 +581,10 @@ public class PdfJsonFallbackFontService { Character.UnicodeScript script = Character.UnicodeScript.of(codePoint); return switch (script) { - // HAN script is used by both Simplified and Traditional Chinese - // Default to Simplified (mainland China, 1.4B speakers) as it's more common - // Traditional Chinese PDFs are detected via font name aliases (MingLiU, PMingLiU, etc.) + // HAN script is used by both Simplified and Traditional Chinese + // Default to Simplified (mainland China, 1.4B speakers) as it's more common + // Traditional Chinese PDFs are detected via font name aliases (MingLiU, PMingLiU, + // etc.) case HAN -> FALLBACK_FONT_CJK_ID; case HIRAGANA, KATAKANA -> FALLBACK_FONT_JP_ID; case HANGUL -> FALLBACK_FONT_KR_ID; diff --git a/app/core/src/test/java/stirling/software/SPDF/model/ApiEndpointTest.java b/app/core/src/test/java/stirling/software/SPDF/model/ApiEndpointTest.java index a9293f1161..bba0720b9f 100644 --- a/app/core/src/test/java/stirling/software/SPDF/model/ApiEndpointTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/model/ApiEndpointTest.java @@ -21,8 +21,7 @@ class ApiEndpointTest { return postNodeWithParams(description, true, names); } - private JsonNode postNodeWithParams( - String description, boolean required, String... names) { + private JsonNode postNodeWithParams(String description, boolean required, String... names) { ObjectNode post = mapper.createObjectNode(); post.put("description", description); ArrayNode params = mapper.createArrayNode(); diff --git a/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceBasicTest.java b/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceBasicTest.java index 79b23f8076..ddce2b3422 100644 --- a/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceBasicTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceBasicTest.java @@ -103,7 +103,9 @@ class LanguageServiceBasicTest { // Verify filtering by restrictions assertTrue(supportedLanguages.contains("en_US"), "Allowed language should be included"); assertTrue(supportedLanguages.contains("fr_FR"), "Allowed language should be included"); - assertFalse(supportedLanguages.contains("en_GB"), "en_GB should NOT be included when not in whitelist"); + assertFalse( + supportedLanguages.contains("en_GB"), + "en_GB should NOT be included when not in whitelist"); assertFalse(supportedLanguages.contains("de_DE"), "Restricted language should be excluded"); } diff --git a/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceTest.java b/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceTest.java index 734f7589e6..a0d235e70d 100644 --- a/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/service/LanguageServiceTest.java @@ -84,11 +84,13 @@ class LanguageServiceTest { // Verify assertEquals( - allowedLanguages, - supportedLanguages, - "Should return only whitelisted languages"); - assertFalse(supportedLanguages.contains("en_GB"), "en_GB should NOT be included when not in whitelist"); - assertFalse(supportedLanguages.contains("de_DE"), "de_DE should NOT be included when not in whitelist"); + allowedLanguages, supportedLanguages, "Should return only whitelisted languages"); + assertFalse( + supportedLanguages.contains("en_GB"), + "en_GB should NOT be included when not in whitelist"); + assertFalse( + supportedLanguages.contains("de_DE"), + "de_DE should NOT be included when not in whitelist"); } @Test diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/EmailService.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/EmailService.java index 5c1cf75450..0d35e8bf34 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/EmailService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/EmailService.java @@ -161,7 +161,7 @@ public class EmailService { String subject = "Welcome to Stirling PDF"; String body = - """ + """
@@ -220,7 +220,7 @@ public class EmailService { String subject = "You've been invited to Stirling PDF"; String body = - """ + """
@@ -269,7 +269,8 @@ public class EmailService { String passwordSection = newPassword == null ? "" - : """ + : + """

Temporary Password: %s

@@ -277,7 +278,7 @@ public class EmailService { .formatted(newPassword); String body = - """ + """
diff --git a/frontend/src/core/components/smartFolders/CardExpansionModal.tsx b/frontend/src/core/components/smartFolders/CardExpansionModal.tsx index da56dba391..ecf20b4eb8 100644 --- a/frontend/src/core/components/smartFolders/CardExpansionModal.tsx +++ b/frontend/src/core/components/smartFolders/CardExpansionModal.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useState, useEffect } from 'react'; import { createPortal } from 'react-dom'; import { Text, ActionIcon, ScrollArea } from '@mantine/core'; import { CardModalPhase, CARD_MODAL_TIMINGS } from '@app/hooks/useCardModalAnimation'; @@ -40,14 +40,23 @@ export function CardExpansionModal({ children, footer, }: CardExpansionModalProps) { + const [viewportW, setViewportW] = useState(window.innerWidth); + const [viewportH, setViewportH] = useState(window.innerHeight); + + useEffect(() => { + const handler = () => { setViewportW(window.innerWidth); setViewportH(window.innerHeight); }; + window.addEventListener('resize', handler); + return () => window.removeEventListener('resize', handler); + }, []); + if (phase === 'closed' || !cardRect) return null; const rootFontSize = parseFloat(getComputedStyle(document.documentElement).fontSize); - const modalW = Math.min(MODAL_W_REM * rootFontSize, window.innerWidth * 0.9); - const modalH = MODAL_H_REM * rootFontSize; + const modalW = Math.min(MODAL_W_REM * rootFontSize, viewportW * 0.9); + const modalH = Math.min(MODAL_H_REM * rootFontSize, viewportH * 0.85); const headerH = HEADER_H_REM * rootFontSize; - const finalLeft = (window.innerWidth - modalW) / 2; - const finalTop = window.innerHeight * MODAL_TOP_FRACTION; + const finalLeft = (viewportW - modalW) / 2; + const finalTop = Math.min(viewportH * MODAL_TOP_FRACTION, viewportH - modalH - 16); const isAtCard = phase === 'entering' || phase === 'closing-header'; const isAtHeader = phase === 'header-open'; diff --git a/frontend/src/core/components/smartFolders/FilePreviewModal.tsx b/frontend/src/core/components/smartFolders/FilePreviewModal.tsx index 236cf6030a..80ff9677fb 100644 --- a/frontend/src/core/components/smartFolders/FilePreviewModal.tsx +++ b/frontend/src/core/components/smartFolders/FilePreviewModal.tsx @@ -1,5 +1,5 @@ import { useEffect, useState } from 'react'; -import { Modal, Center, Text, Box } from '@mantine/core'; +import { Modal, Center, Text, Box, Loader } from '@mantine/core'; import { FileId } from '@app/types/fileContext'; import { fileStorage } from '@app/services/fileStorage'; import { LocalEmbedPDF } from '@app/components/viewer/LocalEmbedPDF'; @@ -14,15 +14,20 @@ interface FilePreviewModalProps { export function FilePreviewModal({ fileId, fileName, onClose }: FilePreviewModalProps) { const [file, setFile] = useState(null); + const [loading, setLoading] = useState(false); const [error, setError] = useState(false); useEffect(() => { - if (!fileId) { setFile(null); setError(false); return; } + if (!fileId) { setFile(null); setError(false); setLoading(false); return; } setError(false); - fileStorage.getStirlingFile(fileId).then(f => { - if (f) setFile(f); - else setError(true); - }); + setLoading(true); + fileStorage.getStirlingFile(fileId) + .then(f => { + if (f) setFile(f); + else setError(true); + }) + .catch(() => setError(true)) + .finally(() => setLoading(false)); }, [fileId]); return ( @@ -34,15 +39,19 @@ export function FilePreviewModal({ fileId, fileName, onClose }: FilePreviewModal zIndex={400} styles={{ body: { height: '82vh', padding: 0, overflow: 'hidden', display: 'flex', flexDirection: 'column' } }} > - {error ? ( + {loading ? ( +
+ ) : error ? (
Could not load file preview.
+ ) : !file ? ( +
) : ( - + )} diff --git a/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx b/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx index fed66ade72..35e70f95b2 100644 --- a/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx +++ b/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx @@ -569,6 +569,11 @@ export function SmartFolderWorkbenchView({ data }: SmartFolderWorkbenchViewProps {status === 'error' && meta?.nextRetryAt && } {status === 'pending' && } {filename} + {status === 'error' && (meta?.failedAttempts ?? 0) > 0 && ( + + {meta!.failedAttempts}Ɨ + + )} {meta?.nextRetryAt && ( )} @@ -635,7 +640,9 @@ export function SmartFolderWorkbenchView({ data }: SmartFolderWorkbenchViewProps {t('smartFolders.workbench.noInputFiles', 'No input files stored yet')} - ) : inputFiles.map((file) => ( + ) : inputFiles.map((file) => { + const inputMeta = folderRecord?.files[file.fileId]; + return ( {file.name} + {inputMeta?.addedAt && {timeAgo(new Date(inputMeta.addedAt), t)}}
- ))} + ); + })} @@ -689,7 +698,11 @@ export function SmartFolderWorkbenchView({ data }: SmartFolderWorkbenchViewProps {t('smartFolders.workbench.noOutputFiles', 'No output files stored yet')} - ) : outputFiles.map((file) => ( + ) : outputFiles.map((file) => { + const processedAt = Object.values(folderRecord?.files ?? {}).find(m => + m.displayFileIds?.includes(file.fileId) || m.displayFileId === file.fileId + )?.processedAt; + return ( {file.name} + {processedAt && {timeAgo(new Date(processedAt), t)}}
- ))} + ); + })} @@ -775,6 +790,7 @@ export function SmartFolderWorkbenchView({ data }: SmartFolderWorkbenchViewProps {filename} + {meta?.lastFailedAt && {timeAgo(new Date(meta.lastFailedAt), t)}} {attempts > 0 && ( 0; - const hasBottomRow = !!file.size || hasTags; + const hasBottomRow = file.size > 0 || otherFolders.length > 0; const handleRowClick = (e: React.MouseEvent) => { if ((e.target as HTMLElement).closest('[data-no-select]')) return; @@ -319,7 +319,7 @@ function FileRow({ paddingLeft: '1.125rem', }} > - {file.size && ( + {file.size > 0 && ( { + for (const file of files) { + const fileId = createFileId(); + const stub: StirlingFileStub = { + id: fileId, + name: file.name, + type: file.type || 'application/pdf', + size: file.size, + lastModified: file.lastModified, + isLeaf: true, + originalFileId: fileId, + versionNumber: 1, + toolHistory: [], + quickKey: createQuickKey(file), + createdAt: Date.now(), + }; + await fileStorage.storeStirlingFile(createStirlingFile(file, fileId), stub); + } + }, []); const handleUploadClick = useCallback(async () => { const pickedFiles = await openFilesFromDisk({ multiple: true, onFallbackOpen: () => uploadInputRef.current?.click(), }); - if (pickedFiles.length > 0) await addFiles(pickedFiles); - }, [addFiles]); + if (pickedFiles.length > 0) await storeFilesOnly(pickedFiles); + }, [storeFilesOnly]); const handleInputChange = useCallback(async (e: React.ChangeEvent) => { const picked = Array.from(e.target.files ?? []); - if (picked.length > 0) await addFiles(picked); + if (picked.length > 0) await storeFilesOnly(picked); e.target.value = ''; - }, [addFiles]); + }, [storeFilesOnly]); const [search, setSearch] = useState(''); const [sortKey, setSortKey] = useState('date-desc'); diff --git a/frontend/src/core/hooks/useFolderAutomation.ts b/frontend/src/core/hooks/useFolderAutomation.ts index 5a5abe6073..7482da6550 100644 --- a/frontend/src/core/hooks/useFolderAutomation.ts +++ b/frontend/src/core/hooks/useFolderAutomation.ts @@ -14,7 +14,7 @@ import { fileStorage } from '@app/services/fileStorage'; import { folderRunStateStorage } from '@app/services/folderRunStateStorage'; import { folderRetryScheduleStorage } from '@app/services/folderRetryScheduleStorage'; import { smartFolderStorage } from '@app/services/smartFolderStorage'; -import { executeAutomationSequence } from '@app/utils/automationExecutor'; +import { executeBackendPipeline } from '@app/utils/automationExecutor'; import { FileId, StirlingFileStub, @@ -101,16 +101,7 @@ export function useFolderAutomation(toolRegistry: Partial) { await folderStorage.updateFileMetadata(folder.id, inputFileId, { status: 'processing' }); - // Step-level callbacks not needed for background folder processing - const noop = () => {}; - const resultFiles = await executeAutomationSequence( - automation, - [file], - toolRegistry as ToolRegistry, - noop, - noop, - noop - ); + const resultFiles = await executeBackendPipeline(automation, [file], toolRegistry as ToolRegistry); // Load input stub for version chain info and name fallback const inputStub = await fileStorage.getStirlingFileStub(inputFileId as FileId); @@ -197,6 +188,7 @@ export function useFolderAutomation(toolRegistry: Partial) { errorMessage: err instanceof Error ? err.message : 'Unknown error', failedAttempts: attempts, nextRetryAt, + lastFailedAt: new Date(), }); if (willRetry) { diff --git a/frontend/src/core/types/smartFolders.ts b/frontend/src/core/types/smartFolders.ts index c20c2a62ee..521b7d0edb 100644 --- a/frontend/src/core/types/smartFolders.ts +++ b/frontend/src/core/types/smartFolders.ts @@ -35,6 +35,7 @@ export interface FolderFileMetadata { errorMessage?: string; failedAttempts?: number; nextRetryAt?: number; // ms timestamp ā€” set when an automatic retry is scheduled + lastFailedAt?: Date; name?: string; // original filename } diff --git a/frontend/src/core/utils/automationExecutor.ts b/frontend/src/core/utils/automationExecutor.ts index 24679bdb71..3286d6e4af 100644 --- a/frontend/src/core/utils/automationExecutor.ts +++ b/frontend/src/core/utils/automationExecutor.ts @@ -5,6 +5,7 @@ import { AUTOMATION_CONSTANTS } from '@app/constants/automation'; import { AutomationFileProcessor } from '@app/utils/automationFileProcessor'; import { ToolType } from '@app/hooks/tools/shared/useToolOperation'; import { processResponse } from '@app/utils/toolResponseProcessor'; +import { getFilenameFromHeaders } from '@app/utils/fileResponseUtils'; /** * Process multi-file tool response (handles ZIP or single PDF responses) @@ -232,3 +233,85 @@ export const executeAutomationSequence = async ( console.log(`\nšŸŽ‰ Automation complete: ${currentFiles.length} file(s)`); return currentFiles; }; + +/** + * Execute an automation pipeline via POST /api/v1/pipeline/handleData. + * + * Falls back to executeAutomationSequence for automations that contain a step requiring + * client-side processing (e.g. Adjust Contrast, Remove Annotations, Extract Pages). + */ +export const executeBackendPipeline = async ( + automation: any, + initialFiles: File[], + toolRegistry: ToolRegistry +): Promise => { + if (!automation?.operations || automation.operations.length === 0) { + throw new Error('No operations in automation'); + } + + // Fall back to frontend execution if any step needs client-side processing + const needsFrontendFallback = automation.operations.some((op: any) => + toolRegistry[op.operation as ToolId]?.operationConfig?.customProcessor != null + ); + if (needsFrontendFallback) { + return executeAutomationSequence(automation, initialFiles, toolRegistry); + } + + // Build PipelineConfig JSON ā€” "pipeline" is the @JsonProperty key the backend expects. + const pipeline = automation.operations.map((op: any) => { + const toolConfig = toolRegistry[op.operation as ToolId]?.operationConfig; + if (!toolConfig) throw new Error(`Tool operation not supported: ${op.operation}`); + + // Apply frontend defaults so the backend receives complete parameters + const parameters = { ...toolConfig.defaultParameters, ...(op.parameters ?? {}) }; + + // Backend builds URL as getBaseUrl() + operation where getBaseUrl() ends with "/" + const rawEndpoint = typeof toolConfig.endpoint === 'function' + ? toolConfig.endpoint(parameters) + : toolConfig.endpoint; + const operation = rawEndpoint.replace(/^\//, ''); + + return { operation, parameters }; + }); + + const formData = new FormData(); + for (const file of initialFiles) { + formData.append('fileInput', file); + } + formData.append('json', JSON.stringify({ name: automation.name, pipeline })); + + const response = await apiClient.post('/api/v1/pipeline/handleData', formData, { + responseType: 'blob', + // Allow per-step timeout headroom proportional to the number of operations + timeout: AUTOMATION_CONSTANTS.OPERATION_TIMEOUT * automation.operations.length, + }); + + const blob: Blob = response.data; + + // Validate the response is an actual PDF or ZIP before storing it. + // An empty or XML/HTML error body from the backend would otherwise be + // silently stored and render as "unknown length" in the PDF viewer. + if (blob.size === 0) { + throw new Error('Backend pipeline returned an empty response'); + } + const header = new Uint8Array(await blob.slice(0, 5).arrayBuffer()); + const isPdf = header[0] === 0x25 && header[1] === 0x50 && header[2] === 0x44 && header[3] === 0x46 && header[4] === 0x2D; // %PDF- + const isZip = header[0] === 0x50 && header[1] === 0x4B; // PK + if (!isPdf && !isZip) { + let hint = ''; + try { hint = ` Response preview: ${await blob.slice(0, 200).text()}`; } catch { /* ignore */ } + throw new Error(`Backend pipeline returned unexpected content (not a PDF or ZIP).${hint}`); + } + + const contentType: string = response.headers['content-type'] ?? ''; + + if (contentType.includes('zip')) { + const { files } = await AutomationFileProcessor.extractAutomationZipFiles(blob); + return files; + } + + const filename = + getFilenameFromHeaders(response.headers['content-disposition'] ?? '') ?? + `${automation.name ?? 'output'}.pdf`; + return [new File([blob], filename, { type: blob.type || 'application/pdf', lastModified: Date.now() })]; +};