From 0f7dbc0993172c3bfdd2f6e66acabcecea0fc8be Mon Sep 17 00:00:00 2001 From: Reece Date: Thu, 30 Apr 2026 22:23:00 +0100 Subject: [PATCH] Improved database --- .../controller/api/WatchFolderController.java | 15 ++ .../proprietary/model/WatchFolder.java | 8 +- .../model/watchfolder/FolderScope.java | 4 +- .../model/watchfolder/InputSource.java | 6 +- .../model/watchfolder/OutputMode.java | 4 +- .../model/watchfolder/ProcessingMode.java | 4 +- .../repository/WatchFolderFileRepository.java | 6 + .../service/WatchFolderService.java | 23 ++- .../smartFolders/SmartFolderHomePage.tsx | 27 +-- .../SmartFolderManagementModal.tsx | 17 +- .../smartFolders/SmartFolderSection.tsx | 4 +- .../smartFolders/SmartFolderWorkbenchView.tsx | 9 +- .../smartFolders/SmartFoldersRegistration.tsx | 6 +- .../contexts/WatchFolderStorageContext.tsx | 30 ++- frontend/src/core/data/smartFolderPresets.ts | 6 +- frontend/src/core/hooks/useAllSmartFolders.ts | 16 +- .../src/core/hooks/useFolderAutomation.ts | 190 +++++++++++------- frontend/src/core/hooks/useFolderData.ts | 38 ++-- .../src/core/hooks/useFolderMembership.ts | 7 +- frontend/src/core/hooks/useFolderOutputIds.ts | 7 +- frontend/src/core/hooks/useFolderRunState.ts | 24 +-- .../src/core/hooks/useFolderRunStatuses.ts | 18 +- .../src/core/hooks/useLocalFolderPoller.ts | 12 +- frontend/src/core/hooks/useSmartFolders.ts | 66 ++---- frontend/src/core/services/folderStorage.ts | 11 +- .../core/services/watchFolderIdbBackend.ts | 2 +- frontend/src/core/types/smartFolders.ts | 27 ++- .../components/WatchFolderServerProvider.tsx | 98 ++++++++- .../services/watchFolderApiService.ts | 8 + .../services/watchFolderServerBackend.ts | 147 ++++++++++---- 30 files changed, 540 insertions(+), 300 deletions(-) diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/WatchFolderController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/WatchFolderController.java index fd624b3add..79692d2900 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/WatchFolderController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/WatchFolderController.java @@ -78,6 +78,15 @@ public class WatchFolderController { return ResponseEntity.noContent().build(); } + @DeleteMapping("/{folderId}/files/{fileId}") + public ResponseEntity deleteFile( + @PathVariable String folderId, @PathVariable String fileId) { + service.deleteFile(folderId, fileId); + // Return 204 whether the row existed or not — DELETE is idempotent and the caller's + // intent ("ensure this row is gone") is satisfied either way. + return ResponseEntity.noContent().build(); + } + // ── Folder runs ──────────────────────────────────────────────────────── @GetMapping("/{folderId}/runs") @@ -97,4 +106,10 @@ public class WatchFolderController { @Valid @RequestBody @Size(max = RUNS_BATCH_MAX) List runs) { return ResponseEntity.status(HttpStatus.CREATED).body(service.addRuns(folderId, runs)); } + + @DeleteMapping("/{folderId}/runs") + public ResponseEntity deleteRuns(@PathVariable String folderId) { + service.deleteRuns(folderId); + return ResponseEntity.noContent().build(); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/WatchFolder.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/WatchFolder.java index c56211af21..394279883f 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/WatchFolder.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/WatchFolder.java @@ -41,10 +41,10 @@ public class WatchFolder implements Serializable { /** * Client-supplied identifier (matches the IndexedDB folder id used by the React frontend). - * Using a String PK — rather than an IDENTITY {@code Long} like other entities — keeps the - * same opaque id across the local IDB cache and the server, so the frontend can round-trip a - * folder between offline/local mode and server mode without remapping. Callers are expected - * to supply a UUID (or equivalently collision-resistant string). + * Using a String PK — rather than an IDENTITY {@code Long} like other entities — keeps the same + * opaque id across the local IDB cache and the server, so the frontend can round-trip a folder + * between offline/local mode and server mode without remapping. Callers are expected to supply + * a UUID (or equivalently collision-resistant string). */ @Id @EqualsAndHashCode.Include diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/FolderScope.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/FolderScope.java index cfabda3aca..11d703b880 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/FolderScope.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/FolderScope.java @@ -10,9 +10,9 @@ import jakarta.persistence.Converter; * Visibility scope of a {@code WatchFolder}. * *
    - *
  • {@link #PERSONAL} — only the owner can see / modify the folder.
    • + *
  • {@link #PERSONAL} — only the owner can see / modify the folder. *
  • {@link #ORGANISATION} — visible to every authenticated user; only admins may create or - * modify.
    • + * modify. *
*/ public enum FolderScope { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/InputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/InputSource.java index 4db75fdbc1..031dfe73ca 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/InputSource.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/InputSource.java @@ -10,9 +10,9 @@ import jakarta.persistence.Converter; * Where input files for a watch folder come from. * *
    - *
  • {@link #IDB} — files dropped / picked in the browser, stored in IndexedDB.
    • - *
  • {@link #LOCAL_FOLDER} — a real folder on the user's machine (desktop build).
    • - *
  • {@link #SERVER_FOLDER} — a directory watched on the server.
    • + *
  • {@link #IDB} — files dropped / picked in the browser, stored in IndexedDB. + *
  • {@link #LOCAL_FOLDER} — a real folder on the user's machine (desktop build). + *
  • {@link #SERVER_FOLDER} — a directory watched on the server. *
*/ public enum InputSource { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/OutputMode.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/OutputMode.java index 8c56c6bce2..dbbfe7e459 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/OutputMode.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/OutputMode.java @@ -10,9 +10,9 @@ import jakarta.persistence.Converter; * How automation output files are produced. * *
    - *
  • {@link #NEW_FILE} — always produce a new, separately-named file.
    • + *
  • {@link #NEW_FILE} — always produce a new, separately-named file. *
  • {@link #NEW_VERSION} — produce a new version of the input file (replacing / versioning - * semantics handled client-side).
    • + * semantics handled client-side). *
*/ public enum OutputMode { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/ProcessingMode.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/ProcessingMode.java index 99990b7725..c27e278d86 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/ProcessingMode.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/ProcessingMode.java @@ -10,9 +10,9 @@ import jakarta.persistence.Converter; * Where the automation pipeline runs. * *
    - *
  • {@link #LOCAL} — runs entirely in the user's browser.
    • + *
  • {@link #LOCAL} — runs entirely in the user's browser. *
  • {@link #SERVER} — runs on the server (forced when {@link InputSource#SERVER_FOLDER} is in - * use).
    • + * use). *
*/ public enum ProcessingMode { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/repository/WatchFolderFileRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/repository/WatchFolderFileRepository.java index 2e580da367..d2d7824b1b 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/repository/WatchFolderFileRepository.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/repository/WatchFolderFileRepository.java @@ -23,4 +23,10 @@ public interface WatchFolderFileRepository extends JpaRepository new IllegalArgumentException("Folder not found: " + id)); + .orElseThrow(() -> new IllegalArgumentException("Folder not found: " + id)); requireWriteAccess(existing); existing.setName(updates.getName()); @@ -114,8 +113,7 @@ public class WatchFolderService { WatchFolder folder = folderRepo .findById(id) - .orElseThrow( - () -> new IllegalArgumentException("Folder not found: " + id)); + .orElseThrow(() -> new IllegalArgumentException("Folder not found: " + id)); requireWriteAccess(folder); // Don't rely on CascadeType.ALL to remove children one-row-at-a-time — for a folder with @@ -196,6 +194,16 @@ public class WatchFolderService { fileRepo.deleteAllByFolderId(folderId); } + /** + * Remove a single file row from a folder. Returns true if a row was deleted, false if no + * matching row existed (already gone, or never present). Idempotent — calling twice is safe. + */ + @Transactional + public boolean deleteFile(String folderId, String fileId) { + requireWriteAccess(folderId); + return fileRepo.deleteByFolderIdAndFileId(folderId, fileId) > 0; + } + // ── Folder runs ──────────────────────────────────────────────────────── @Transactional(readOnly = true) @@ -229,6 +237,13 @@ public class WatchFolderService { return runRepo.saveAll(runs); } + /** Bulk-delete all runs for a folder. */ + @Transactional + public void deleteRuns(String folderId) { + requireWriteAccess(folderId); + runRepo.deleteAllByFolderId(folderId); + } + // ── Auth helpers ─────────────────────────────────────────────────────── private User currentUser() { diff --git a/frontend/src/core/components/smartFolders/SmartFolderHomePage.tsx b/frontend/src/core/components/smartFolders/SmartFolderHomePage.tsx index 7a2498ab13..cc52ababca 100644 --- a/frontend/src/core/components/smartFolders/SmartFolderHomePage.tsx +++ b/frontend/src/core/components/smartFolders/SmartFolderHomePage.tsx @@ -12,13 +12,13 @@ import PauseCircleOutlineIcon from "@mui/icons-material/PauseCircleOutline"; import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline"; import { useSmartFolders } from "@app/hooks/useSmartFolders"; import { useFolderRunStatuses } from "@app/hooks/useFolderRunStatuses"; -import { useFolderAutomation, resolveInputFile } from "@app/hooks/useFolderAutomation"; +import { useFolderAutomation, resolveInputFile, resolveFolderAutomation } from "@app/hooks/useFolderAutomation"; import { SmartFolder } from "@app/types/smartFolders"; import { AutomationConfig } from "@app/types/automation"; import { iconMap } from "@app/components/tools/automate/iconMap"; -import { automationStorage } from "@app/services/automationStorage"; import { folderStorage } from "@app/services/folderStorage"; import { fileStorage } from "@app/services/fileStorage"; +import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext"; import { SmartFolderManagementModal } from "@app/components/smartFolders/SmartFolderManagementModal"; import { DeleteFolderConfirmModal } from "@app/components/smartFolders/DeleteFolderConfirmModal"; import { useToolWorkflow } from "@app/contexts/ToolWorkflowContext"; @@ -66,6 +66,7 @@ function FolderCard({ onTogglePause, }: FolderCardProps) { const { t } = useTranslation(); + const store = useWatchFolderStore(); const [automation, setAutomation] = useState(null); const [fileCount, setFileCount] = useState(0); const [lastAdded, setLastAdded] = useState(null); @@ -73,10 +74,10 @@ function FolderCard({ const [isHovered, setIsHovered] = useState(false); useEffect(() => { - automationStorage.getAutomation(folder.automationId).then(setAutomation); + resolveFolderAutomation(folder).then(setAutomation); const loadData = () => - folderStorage.getFolderData(folder.id).then((record) => { + store.getFolderData(folder.id).then((record) => { if (!record) { setFileCount(0); setLastAdded(null); @@ -89,11 +90,12 @@ function FolderCard({ }); loadData(); + // Server backend mirrors writes to IDB so this fires for both backends. const unsub = folderStorage.onFolderChange((changedId) => { if (changedId === folder.id) loadData(); }); return unsub; - }, [folder.id, folder.automationId]); + }, [folder.id, folder.automationId, folder.automationConfig, store]); const FolderIcon = iconMap[folder.icon as keyof typeof iconMap] ?? iconMap.FolderIcon; const isPaused = folder.isPaused ?? false; @@ -488,6 +490,7 @@ export function SmartFolderHomePage() { const { toolRegistry, setCustomWorkbenchViewData } = useToolWorkflow(); const { actions } = useNavigationActions(); const { processBatch } = useFolderAutomation(toolRegistry); + const store = useWatchFolderStore(); const [createModalOpen, setCreateModalOpen] = useState(false); const [editFolder, setEditFolder] = useState(null); @@ -505,7 +508,7 @@ export function SmartFolderHomePage() { const handleEdit = useCallback(async (folder: SmartFolder) => { setEditFolder(folder); - const automation = await automationStorage.getAutomation(folder.automationId); + const automation = await resolveFolderAutomation(folder); setEditAutomation(automation); setCreateModalOpen(true); }, []); @@ -522,7 +525,7 @@ export function SmartFolderHomePage() { if (pdfs.length === 0) return; // Load existing folder data once to detect re-runs (same sidebar file dropped again) - const existingData = await folderStorage.getFolderData(folder.id); + const existingData = await store.getFolderData(folder.id); // Register files sequentially — addFileToFolder/updateFileMetadata are read-modify-write // without IDB transactions, so concurrent calls on the same folder lose updates. const items: Array<{ file: File; inputFileId: string; ownedByFolder: boolean }> = []; @@ -530,9 +533,9 @@ export function SmartFolderHomePage() { const { inputFileId, ownedByFolder } = await resolveInputFile(file); if (existingData?.files[inputFileId]) { // Re-processing: preserve addedAt and ownedByFolder, just reset status - await folderStorage.updateFileMetadata(folder.id, inputFileId, { status: "pending", errorMessage: undefined }); + await store.updateFileMetadata(folder.id, inputFileId, { status: "pending", errorMessage: undefined }); } else { - await folderStorage.addFileToFolder(folder.id, inputFileId, { + await store.addFileToFolder(folder.id, inputFileId, { status: "pending", name: file.name, ownedByFolder: ownedByFolder || undefined, @@ -557,7 +560,7 @@ export function SmartFolderHomePage() { }); } }, - [processBatch], + [processBatch, store], ); const handleTogglePause = useCallback( @@ -569,7 +572,7 @@ export function SmartFolderHomePage() { if (resuming) { // Process any files that were queued while paused - const record = await folderStorage.getFolderData(folder.id); + const record = await store.getFolderData(folder.id); if (record) { const pendingEntries = Object.entries(record.files).filter(([, meta]) => meta.status === "pending"); if (pendingEntries.length > 0) { @@ -585,7 +588,7 @@ export function SmartFolderHomePage() { } } }, - [updateFolder, refreshFolders, processFiles], + [updateFolder, refreshFolders, processBatch, store], ); const handleDropSidebarFile = useCallback( diff --git a/frontend/src/core/components/smartFolders/SmartFolderManagementModal.tsx b/frontend/src/core/components/smartFolders/SmartFolderManagementModal.tsx index e0ee582dba..179997de9a 100644 --- a/frontend/src/core/components/smartFolders/SmartFolderManagementModal.tsx +++ b/frontend/src/core/components/smartFolders/SmartFolderManagementModal.tsx @@ -21,7 +21,7 @@ import { AutomationConfig, AutomationMode } from '@app/types/automation'; import { IconPicker as IconSelector } from '@app/components/smartFolders/IconPicker'; import AutomationCreation from '@app/components/tools/automate/AutomationCreation'; import { useToolWorkflow } from '@app/contexts/ToolWorkflowContext'; -import { smartFolderStorage } from '@app/services/smartFolderStorage'; +import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext'; import { folderDirectoryHandleStorage } from '@app/services/folderDirectoryHandleStorage'; import { createServerFolder, @@ -75,6 +75,7 @@ export function SmartFolderManagementModal({ }: SmartFolderManagementModalProps) { const { t } = useTranslation(); const { toolRegistry } = useToolWorkflow(); + const store = useWatchFolderStore(); const isEditMode = !!editFolder; // Animation state @@ -186,12 +187,20 @@ export function SmartFolderManagementModal({ const retryFields = { maxRetries, retryDelayMinutes }; const hasOutputDirectory = outputDirName !== null; const ttlHoursNum = isServerFolder && outputTtlHours !== 'forever' ? Number(outputTtlHours) : null; + // Inline the automation config when persisting to a server-backed store — + // server folders use this instead of looking up automationId in browser IDB. + const inlinedAutomationConfig = JSON.stringify({ + name: automation.name, + description: automation.description, + operations: automation.operations, + }); const folderData = { name: trimmedName, description: '', icon, accentColor, automationId: automation.id, + automationConfig: inlinedAutomationConfig, ...retryFields, outputMode: outputMode === 'new_version' ? 'new_version' as const : undefined, outputName: outputName.trim() || undefined, @@ -206,7 +215,7 @@ export function SmartFolderManagementModal({ if (isEditMode && editFolder) { const wasServerFolder = editFolder.inputSource === 'server-folder'; const wasLocalFolder = editFolder.inputSource === 'local-folder'; - await smartFolderStorage.updateFolder({ ...editFolder, ...folderData }); + await store.updateFolder({ ...editFolder, ...folderData }); if (pendingDirHandle.current) { await folderDirectoryHandleStorage.set(editFolder.id, pendingDirHandle.current); } else if (!hasOutputDirectory) { @@ -229,7 +238,7 @@ export function SmartFolderManagementModal({ await deleteServerFolder(editFolder.id).catch(() => {}); } } else { - const newFolder = await smartFolderStorage.createFolder(folderData); + const newFolder = await store.createFolder(folderData); if (pendingDirHandle.current) { await folderDirectoryHandleStorage.set(newFolder.id, pendingDirHandle.current); } @@ -249,7 +258,7 @@ export function SmartFolderManagementModal({ } finally { setSaving(false); } - }, [name, icon, accentColor, outputMode, outputName, outputNamePosition, outputDirName, maxRetries, retryDelayMinutes, inputSource, outputTtlHours, deleteOutputOnDownload, isEditMode, editFolder, toolRegistry, resetState, onSaved, onClose, t]); + }, [name, icon, accentColor, outputMode, outputName, outputNamePosition, outputDirName, maxRetries, retryDelayMinutes, inputSource, outputTtlHours, deleteOutputOnDownload, isEditMode, editFolder, toolRegistry, resetState, onSaved, onClose, t, store]); const handleSave = () => { const trimmedName = name.trim(); diff --git a/frontend/src/core/components/smartFolders/SmartFolderSection.tsx b/frontend/src/core/components/smartFolders/SmartFolderSection.tsx index d4d6ddcff0..1e5cc1299d 100644 --- a/frontend/src/core/components/smartFolders/SmartFolderSection.tsx +++ b/frontend/src/core/components/smartFolders/SmartFolderSection.tsx @@ -11,7 +11,7 @@ import { DeleteFolderConfirmModal } from '@app/components/smartFolders/DeleteFol import { SmartFolderCard } from '@app/components/smartFolders/SmartFolderCard'; import { SmartFolder } from '@app/types/smartFolders'; import { AutomationConfig } from '@app/types/automation'; -import { automationStorage } from '@app/services/automationStorage'; +import { resolveFolderAutomation } from '@app/hooks/useFolderAutomation'; import { SMART_FOLDER_VIEW_ID, SMART_FOLDER_WORKBENCH_ID } from '@app/components/smartFolders/SmartFoldersRegistration'; export function SmartFolderSection() { @@ -37,7 +37,7 @@ export function SmartFolderSection() { const handleEditFolder = async (e: React.MouseEvent, folder: SmartFolder) => { e.stopPropagation(); setEditFolder(folder); - const automation = await automationStorage.getAutomation(folder.automationId); + const automation = await resolveFolderAutomation(folder); setEditAutomation(automation); setCreateModalOpen(true); }; diff --git a/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx b/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx index dd9bc76fde..74deaf4991 100644 --- a/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx +++ b/frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx @@ -26,8 +26,7 @@ import { useFolderData } from "@app/hooks/useFolderData"; import { useFolderRunState } from "@app/hooks/useFolderRunState"; import { useToolWorkflow } from "@app/contexts/ToolWorkflowContext"; import { SMART_FOLDER_VIEW_ID, SMART_FOLDER_WORKBENCH_ID } from "@app/components/smartFolders/SmartFoldersRegistration"; -import { automationStorage } from "@app/services/automationStorage"; -import { useFolderAutomation, resolveInputFile } from "@app/hooks/useFolderAutomation"; +import { useFolderAutomation, resolveInputFile, resolveFolderAutomation } from "@app/hooks/useFolderAutomation"; import { useLocalFolderPoller } from "@app/hooks/useLocalFolderPoller"; import { AutomationConfig } from "@app/types/automation"; import { iconMap } from "@app/components/tools/automate/iconMap"; @@ -184,10 +183,10 @@ export function SmartFolderWorkbenchView({ data }: SmartFolderWorkbenchViewProps }, [folderRecord, isServerFolder]); useEffect(() => { - if (folder?.automationId) { - automationStorage.getAutomation(folder.automationId).then(setAutomation); + if (folder) { + resolveFolderAutomation(folder).then(setAutomation); } - }, [folder?.automationId]); + }, [folder]); useEffect(() => { if (isLocalFolder && folderId) { diff --git a/frontend/src/core/components/smartFolders/SmartFoldersRegistration.tsx b/frontend/src/core/components/smartFolders/SmartFoldersRegistration.tsx index aa007fe787..2da09845fe 100644 --- a/frontend/src/core/components/smartFolders/SmartFoldersRegistration.tsx +++ b/frontend/src/core/components/smartFolders/SmartFoldersRegistration.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef } from 'react'; import { useTranslation } from 'react-i18next'; import { useToolWorkflow } from '@app/contexts/ToolWorkflowContext'; +import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext'; import { SmartFolderWorkbenchView } from '@app/components/smartFolders/SmartFolderWorkbenchView'; import { seedDefaultFolders } from '@app/data/smartFolderPresets'; import { useWatchFolderUrlSync } from '@app/hooks/useWatchFolderUrlSync'; @@ -11,6 +12,7 @@ export const SMART_FOLDER_WORKBENCH_ID = 'custom:smartFolder' as const; export default function SmartFoldersRegistration() { const { t } = useTranslation(); const { registerCustomWorkbenchView, unregisterCustomWorkbenchView, clearCustomWorkbenchViewData } = useToolWorkflow(); + const store = useWatchFolderStore(); useWatchFolderUrlSync(); // Keep refs to latest cleanup callbacks so the registration effect doesn't @@ -22,8 +24,8 @@ export default function SmartFoldersRegistration() { useEffect(() => { clearRef.current = clearCustomWorkbenchViewData; }); useEffect(() => { - seedDefaultFolders(); - }, []); + seedDefaultFolders(store); + }, [store]); useEffect(() => { registerCustomWorkbenchView({ diff --git a/frontend/src/core/contexts/WatchFolderStorageContext.tsx b/frontend/src/core/contexts/WatchFolderStorageContext.tsx index ae9de1dbd0..6c1d7df42e 100644 --- a/frontend/src/core/contexts/WatchFolderStorageContext.tsx +++ b/frontend/src/core/contexts/WatchFolderStorageContext.tsx @@ -7,10 +7,10 @@ */ import React, { createContext, useContext } from "react"; -import { SmartFolder, FolderRecord } from "@app/types/smartFolders"; -import { SmartFolderRunEntry } from "@app/types/smartFolders"; +import type { SmartFolder, FolderRecord, FolderFileMetadata, SmartFolderRunEntry } from "@app/types/smartFolders"; +import { idbBackend } from "@app/services/watchFolderIdbBackend"; -// ── Storage interface ────────────────────────────────────────────────────── +// ── Storage interface ───────────────────────────────────────────────────── export interface WatchFolderStorageBackend { // Folder CRUD @@ -23,16 +23,9 @@ export interface WatchFolderStorageBackend { // File metadata getFolderData(folderId: string): Promise; - updateFileMetadata( - folderId: string, - fileId: string, - meta: Partial, - ): Promise; - addFileToFolder( - folderId: string, - fileId: string, - meta?: Partial, - ): Promise; + updateFileMetadata(folderId: string, fileId: string, meta: Partial): Promise; + addFileToFolder(folderId: string, fileId: string, meta?: Partial): Promise; + removeFileFromFolder(folderId: string, fileId: string): Promise; clearFolder(folderId: string): Promise; // Run state @@ -44,7 +37,7 @@ export interface WatchFolderStorageBackend { onChange(callback: () => void): () => void; } -// ── Context ──────────────────────────────────────────────────────────────── +// ── Context ─────────────────────────────────────────────────────────────── const WatchFolderStorageContext = createContext(null); @@ -59,9 +52,10 @@ export function WatchFolderStorageProvider({ } /** - * Returns the storage backend from context, or null if none is provided - * (meaning hooks should fall back to direct IDB imports). + * Returns the storage backend, falling back to `idbBackend` if no provider is mounted. + * The provider IS always mounted in the app, so the fallback is purely defensive + * (e.g. tests rendering hooks in isolation). */ -export function useWatchFolderStorage(): WatchFolderStorageBackend | null { - return useContext(WatchFolderStorageContext); +export function useWatchFolderStore(): WatchFolderStorageBackend { + return useContext(WatchFolderStorageContext) ?? idbBackend; } diff --git a/frontend/src/core/data/smartFolderPresets.ts b/frontend/src/core/data/smartFolderPresets.ts index 826f3a042d..9bd407f683 100644 --- a/frontend/src/core/data/smartFolderPresets.ts +++ b/frontend/src/core/data/smartFolderPresets.ts @@ -5,7 +5,7 @@ import { SmartFolder } from "@app/types/smartFolders"; import { AutomationConfig } from "@app/types/automation"; import { automationStorage } from "@app/services/automationStorage"; -import { smartFolderStorage } from "@app/services/smartFolderStorage"; +import type { WatchFolderStorageBackend } from "@app/contexts/WatchFolderStorageContext"; const SEEDED_FLAG = "smart_folders_seeded"; let seedingInProgress = false; @@ -94,14 +94,14 @@ const PRESETS: PresetDefinition[] = [ }, ]; -export async function seedDefaultFolders(): Promise { +export async function seedDefaultFolders(store: WatchFolderStorageBackend): Promise { if (localStorage.getItem(SEEDED_FLAG) || seedingInProgress) return; seedingInProgress = true; try { for (const preset of PRESETS) { const savedAutomation = await automationStorage.saveAutomation(preset.automation); - await smartFolderStorage.createFolder({ + await store.createFolder({ ...preset.folder, automationId: savedAutomation.id, }); diff --git a/frontend/src/core/hooks/useAllSmartFolders.ts b/frontend/src/core/hooks/useAllSmartFolders.ts index 6b64905588..e7ea2dfb02 100644 --- a/frontend/src/core/hooks/useAllSmartFolders.ts +++ b/frontend/src/core/hooks/useAllSmartFolders.ts @@ -6,30 +6,24 @@ import { useState, useEffect } from "react"; import { SmartFolder } from "@app/types/smartFolders"; -import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/services/smartFolderStorage"; -import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext"; +import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext"; export function useAllSmartFolders(): SmartFolder[] { - const backend = useWatchFolderStorage(); + const store = useWatchFolderStore(); const [folders, setFolders] = useState([]); useEffect(() => { const load = async () => { try { - const all = backend ? await backend.getAllFolders() : await smartFolderStorage.getAllFolders(); - setFolders(all); + setFolders(await store.getAllFolders()); } catch (err) { console.error("Failed to load smart folders:", err); } }; load(); - if (backend) { - return backend.onChange(load); - } - window.addEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, load); - return () => window.removeEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, load); - }, [backend]); + return store.onChange(load); + }, [store]); return folders; } diff --git a/frontend/src/core/hooks/useFolderAutomation.ts b/frontend/src/core/hooks/useFolderAutomation.ts index 0a34e710a5..eefc000639 100644 --- a/frontend/src/core/hooks/useFolderAutomation.ts +++ b/frontend/src/core/hooks/useFolderAutomation.ts @@ -13,13 +13,12 @@ import { useCallback, useEffect, useRef } from 'react'; import { addSSEHandler, parsePipelineSSEEvent } from '@app/hooks/useSSEConnection'; import { ToolRegistry } from '@app/data/toolsTaxonomy'; -import { SmartFolder, isServerFolderInput } from '@app/types/smartFolders'; +import { SmartFolder, isServerFolderInput, FolderFileMetadata } from '@app/types/smartFolders'; import { automationStorage } from '@app/services/automationStorage'; -import { folderStorage } from '@app/services/folderStorage'; +import type { AutomationConfig } from '@app/types/automation'; import { fileStorage } from '@app/services/fileStorage'; -import { folderRunStateStorage } from '@app/services/folderRunStateStorage'; import { folderRetryScheduleStorage } from '@app/services/folderRetryScheduleStorage'; -import { smartFolderStorage } from '@app/services/smartFolderStorage'; +import { useWatchFolderStore, WatchFolderStorageBackend } from '@app/contexts/WatchFolderStorageContext'; import { executeBackendPipeline, submitBackendJob, @@ -80,6 +79,37 @@ function notifySW(message: { type: string }): void { navigator.serviceWorker?.controller?.postMessage(message); } +/** + * Resolves the automation to run for a folder. + * + * Server-backed folders carry the pipeline inline as `automationConfig` (the IDB + * automationStorage on this client may not have an entry for `automationId`), so + * prefer that. Otherwise look up by `automationId` in IDB. + */ +export async function resolveFolderAutomation(folder: SmartFolder): Promise { + if (folder.automationConfig) { + try { + const parsed = JSON.parse(folder.automationConfig); + const operations = Array.isArray(parsed) ? parsed : parsed.operations; + if (Array.isArray(operations)) { + return { + id: folder.automationId || `inline:${folder.id}`, + name: parsed.name ?? folder.name, + description: parsed.description ?? '', + icon: folder.icon, + operations, + createdAt: folder.createdAt, + updatedAt: folder.updatedAt, + }; + } + } catch { + // Fall through to IDB lookup + } + } + if (!folder.automationId) return null; + return automationStorage.getAutomation(folder.automationId); +} + // --------------------------------------------------------------------------- // Output finalisation — shared by runPipeline (sync fallback) and drainPendingJobs // --------------------------------------------------------------------------- @@ -90,13 +120,14 @@ function notifySW(message: { type: string }): void { * executeBackendPipeline call or an async job poll. */ async function finalizeRun( + store: WatchFolderStorageBackend, folder: SmartFolder, file: File, inputFileId: string, ownedByFolder: boolean, resultFiles: File[] ): Promise { - const currentFolderData = await folderStorage.getFolderData(folder.id); + const currentFolderData = await store.getFolderData(folder.id); const currentMeta = currentFolderData?.files[inputFileId]; const prevOutputIds: string[] = currentMeta?.displayFileIds ?? (currentMeta?.displayFileId ? [currentMeta.displayFileId] : []); @@ -201,14 +232,14 @@ async function finalizeRun( const processedAt = new Date(); const accumulatedIds = isAutoNumber ? [...prevOutputIds, ...allOutputIds] : allOutputIds; - await folderStorage.updateFileMetadata(folder.id, inputFileId, { + await store.updateFileMetadata(folder.id, inputFileId, { status: 'processed', processedAt, displayFileId: accumulatedIds[0], displayFileIds: accumulatedIds, }); - await folderRunStateStorage.appendRunEntries(folder.id, [{ + await store.addFolderRunEntries(folder.id, [{ inputFileId, displayFileId: accumulatedIds[0], displayFileIds: accumulatedIds, @@ -222,11 +253,12 @@ async function finalizeRun( // --------------------------------------------------------------------------- async function findFileByJobId( + store: WatchFolderStorageBackend, jobId: string -): Promise<{ folder: SmartFolder; fileId: string; meta: import('@app/types/smartFolders').FolderFileMetadata } | null> { - const folders = await smartFolderStorage.getAllFolders(); +): Promise<{ folder: SmartFolder; fileId: string; meta: FolderFileMetadata } | null> { + const folders = await store.getAllFolders(); for (const folder of folders) { - const folderData = await folderStorage.getFolderData(folder.id); + const folderData = await store.getFolderData(folder.id); if (!folderData) continue; for (const [fileId, meta] of Object.entries(folderData.files)) { if (meta.serverJobId === jobId) return { folder, fileId, meta }; @@ -252,22 +284,23 @@ async function findFileByJobId( * executeBackendPipeline (tab-close resilience does not apply to these). */ export function useFolderAutomation(toolRegistry: Partial) { + const store = useWatchFolderStore(); const processingRef = useRef>(new Set()); // ── Finalise a job from the SSE handler ─────────────────────────────────── const finalizeFromSSE = useCallback(async (jobId: string, error?: string) => { - const match = await findFileByJobId(jobId); + const match = await findFileByJobId(store, jobId); if (!match) return; const { folder, fileId, meta } = match; if (processingRef.current.has(fileId)) return; // drain already handling it processingRef.current.add(fileId); try { - const freshMeta = (await folderStorage.getFolderData(folder.id))?.files[fileId]; + const freshMeta = (await store.getFolderData(folder.id))?.files[fileId]; if (freshMeta?.status !== 'processing') return; // already finalised by drain if (error) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: error, serverJobId: undefined, @@ -277,7 +310,7 @@ export function useFolderAutomation(toolRegistry: Partial) { const inputFile = await fileStorage.getStirlingFile(fileId as FileId); if (!inputFile) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: 'Input file missing from storage', serverJobId: undefined, @@ -285,10 +318,10 @@ export function useFolderAutomation(toolRegistry: Partial) { return; } const resultFiles = await getBackendJobResult(jobId, folder.name); - await finalizeRun(folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles); - await folderStorage.updateFileMetadata(folder.id, fileId, { serverJobId: undefined }); + await finalizeRun(store, folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles); + await store.updateFileMetadata(folder.id, fileId, { serverJobId: undefined }); } catch (err) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: err instanceof Error ? err.message : 'Failed to retrieve job result', serverJobId: undefined, @@ -296,7 +329,7 @@ export function useFolderAutomation(toolRegistry: Partial) { } finally { processingRef.current.delete(fileId); } - }, []); + }, [store]); // ── Server-folder SSE completion handler ────────────────────────────────── // Output filenames are "{fileId}.{ext}" — strip extension to get the IDB fileId directly. @@ -306,9 +339,9 @@ export function useFolderAutomation(toolRegistry: Partial) { folderId: string, outputFilenames: string[] ) => { - const folder = await smartFolderStorage.getFolder(folderId); + const folder = await store.getFolder(folderId); if (!folder || folder.isPaused) return; - const folderData = await folderStorage.getFolderData(folderId); + const folderData = await store.getFolderData(folderId); if (!folderData) return; for (const outputFilename of outputFilenames) { @@ -321,19 +354,19 @@ export function useFolderAutomation(toolRegistry: Partial) { if (processingRef.current.has(fileId)) continue; processingRef.current.add(fileId); try { - const freshMeta = (await folderStorage.getFolderData(folderId))?.files[fileId]; + const freshMeta = (await store.getFolderData(folderId))?.files[fileId]; if (freshMeta?.status !== 'processing') continue; // already finalised const processedAt = new Date(); // Record completion — output lives on the server, not in IDB file storage - await folderStorage.updateFileMetadata(folderId, fileId, { + await store.updateFileMetadata(folderId, fileId, { status: 'processed', processedAt, serverOutputFilenames: [outputFilename], pendingOnServerFolder: undefined, }); - await folderRunStateStorage.appendRunEntries(folderId, [{ + await store.addFolderRunEntries(folderId, [{ inputFileId: fileId, displayFileId: fileId, processedAt, @@ -363,7 +396,7 @@ export function useFolderAutomation(toolRegistry: Partial) { } } } catch (err) { - await folderStorage.updateFileMetadata(folderId, fileId, { + await store.updateFileMetadata(folderId, fileId, { status: 'error', errorMessage: err instanceof Error ? err.message : 'Failed to finalize server output', pendingOnServerFolder: undefined, @@ -372,7 +405,7 @@ export function useFolderAutomation(toolRegistry: Partial) { processingRef.current.delete(fileId); } } - }, []); + }, [store]); // ── Server-folder SSE error handler ─────────────────────────────────────── // Marks files as failed when PipelineDirectoryProcessor reports a batch error. @@ -384,7 +417,7 @@ export function useFolderAutomation(toolRegistry: Partial) { if (processingRef.current.has(fileId)) continue; processingRef.current.add(fileId); try { - await folderStorage.updateFileMetadata(folderId, fileId, { + await store.updateFileMetadata(folderId, fileId, { status: 'error', errorMessage: 'Server-side processing failed', pendingOnServerFolder: undefined, @@ -393,17 +426,17 @@ export function useFolderAutomation(toolRegistry: Partial) { processingRef.current.delete(fileId); } } - }, []); + }, [store]); // ── Recovery drain — runs once on mount/visibility/SW wake ──────────────── // Only needed when SSE was down during job completion (tab close, server restart). // On transient network errors: leave as 'processing' — drain retries next cycle. // On 404: job expired from server — mark as error. const drainPendingJobs = useCallback(async () => { - const folders = await smartFolderStorage.getAllFolders(); + const folders = await store.getAllFolders(); for (const folder of folders) { if (folder.isPaused) continue; - const folderData = await folderStorage.getFolderData(folder.id); + const folderData = await store.getFolderData(folder.id); if (!folderData) continue; for (const [fileId, meta] of Object.entries(folderData.files)) { @@ -420,7 +453,7 @@ export function useFolderAutomation(toolRegistry: Partial) { // 404 → job expired (server restarted / TTL hit) — surface as error // Anything else → transient network issue, leave as 'processing' for next drain if ((err as any)?.response?.status === 404) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: 'Job expired — server may have restarted. Retry to reprocess.', serverJobId: undefined, @@ -432,7 +465,7 @@ export function useFolderAutomation(toolRegistry: Partial) { if (jobStatus.status === 'completed') { const inputFile = await fileStorage.getStirlingFile(fileId as FileId); if (!inputFile) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: 'Input file missing from storage', serverJobId: undefined, @@ -441,17 +474,17 @@ export function useFolderAutomation(toolRegistry: Partial) { } try { const resultFiles = await getBackendJobResult(jobId, folder.name); - await finalizeRun(folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles); - await folderStorage.updateFileMetadata(folder.id, fileId, { serverJobId: undefined }); + await finalizeRun(store, folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles); + await store.updateFileMetadata(folder.id, fileId, { serverJobId: undefined }); } catch (err) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: err instanceof Error ? err.message : 'Failed to retrieve job result', serverJobId: undefined, }); } } else if (jobStatus.status === 'failed') { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: jobStatus.error || 'Server job failed', serverJobId: undefined, @@ -485,12 +518,12 @@ export function useFolderAutomation(toolRegistry: Partial) { if (processingRef.current.has(fileId)) continue; processingRef.current.add(fileId); try { - const freshMeta = (await folderStorage.getFolderData(folder.id))?.files[fileId]; + const freshMeta = (await store.getFolderData(folder.id))?.files[fileId]; if (freshMeta?.status !== 'processing') continue; const inputFile = await fileStorage.getStirlingFile(fileId as FileId); if (!inputFile) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: 'Input file missing from storage', pendingOnServerFolder: undefined, @@ -498,13 +531,13 @@ export function useFolderAutomation(toolRegistry: Partial) { continue; } const processedAt = new Date(); - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'processed', processedAt, serverOutputFilenames: [outputFile.filename], pendingOnServerFolder: undefined, }); - await folderRunStateStorage.appendRunEntries(folder.id, [{ + await store.addFolderRunEntries(folder.id, [{ inputFileId: fileId, displayFileId: fileId, processedAt, @@ -529,7 +562,7 @@ export function useFolderAutomation(toolRegistry: Partial) { } catch { /* best-effort */ } } } catch (err) { - await folderStorage.updateFileMetadata(folder.id, fileId, { + await store.updateFileMetadata(folder.id, fileId, { status: 'error', errorMessage: err instanceof Error ? err.message : 'Failed to retrieve server output', pendingOnServerFolder: undefined, @@ -539,7 +572,7 @@ export function useFolderAutomation(toolRegistry: Partial) { } } } - }, []); + }, [store]); // ── Core pipeline runner ─────────────────────────────────────────────────── const runPipeline = useCallback( @@ -553,16 +586,16 @@ export function useFolderAutomation(toolRegistry: Partial) { processingRef.current.add(inputFileId); try { - const automation = await automationStorage.getAutomation(folder.automationId); + const automation = await resolveFolderAutomation(folder); if (!automation) { - await folderStorage.updateFileMetadata(folder.id, inputFileId, { + await store.updateFileMetadata(folder.id, inputFileId, { status: 'error', errorMessage: 'Automation not found', }); return; } - await folderStorage.updateFileMetadata(folder.id, inputFileId, { status: 'processing' }); + await store.updateFileMetadata(folder.id, inputFileId, { status: 'processing' }); // Server-folder input — upload to watch folder, trigger immediate processing via SSE if (isServerFolderInput(folder)) { @@ -571,7 +604,7 @@ export function useFolderAutomation(toolRegistry: Partial) { // is no longer readable after the async resolveInputFile call completed. const uploadFile = await fileStorage.getStirlingFile(inputFileId as FileId) ?? file; await uploadFileToServerFolder(folder.id, inputFileId, uploadFile); - await folderStorage.updateFileMetadata(folder.id, inputFileId, { + await store.updateFileMetadata(folder.id, inputFileId, { pendingOnServerFolder: true, }); // Fire-and-forget trigger — don't wait for processing to start @@ -584,7 +617,7 @@ export function useFolderAutomation(toolRegistry: Partial) { const jobId = await submitBackendJob(automation, [file], toolRegistry); if (jobId !== null) { - await folderStorage.updateFileMetadata(folder.id, inputFileId, { serverJobId: jobId }); + await store.updateFileMetadata(folder.id, inputFileId, { serverJobId: jobId }); // Release lock — SSE handler / drain will re-acquire when finalising processingRef.current.delete(inputFileId); return; @@ -592,10 +625,10 @@ export function useFolderAutomation(toolRegistry: Partial) { // Sync fallback (automation contains a custom-processor step) const resultFiles = await executeBackendPipeline(automation, [file], toolRegistry); - await finalizeRun(folder, file, inputFileId, ownedByFolder, resultFiles); + await finalizeRun(store, folder, file, inputFileId, ownedByFolder, resultFiles); } catch (err: unknown) { - const existing = await folderStorage.getFolderData(folder.id); + const existing = await store.getFolderData(folder.id); const prev = existing?.files[inputFileId]; const attempts = (prev?.failedAttempts ?? 0) + 1; const maxRetries = folder.maxRetries ?? 3; @@ -603,7 +636,7 @@ export function useFolderAutomation(toolRegistry: Partial) { const willRetry = maxRetries > 0 && attempts < maxRetries && retryDelayMs > 0; const nextRetryAt = willRetry ? Date.now() + retryDelayMs : undefined; - await folderStorage.updateFileMetadata(folder.id, inputFileId, { + await store.updateFileMetadata(folder.id, inputFileId, { status: 'error', errorMessage: err instanceof Error ? err.message : 'Unknown error', failedAttempts: attempts, @@ -624,10 +657,10 @@ export function useFolderAutomation(toolRegistry: Partial) { } finally { // Safety net: if still 'processing' without a serverJobId, something went wrong try { - const record = await folderStorage.getFolderData(folder.id); + const record = await store.getFolderData(folder.id); const fileMeta = record?.files[inputFileId]; if (fileMeta?.status === 'processing' && !fileMeta?.serverJobId && !fileMeta?.pendingOnServerFolder) { - await folderStorage.updateFileMetadata(folder.id, inputFileId, { + await store.updateFileMetadata(folder.id, inputFileId, { status: 'error', errorMessage: 'Processing failed unexpectedly', }); @@ -638,14 +671,14 @@ export function useFolderAutomation(toolRegistry: Partial) { processingRef.current.delete(inputFileId); } }, - [toolRegistry] + [toolRegistry, store] ); // ── Sync server-folder sessions on mount ────────────────────────────────── // Ensures the server's session.json always points to the current browser session, // so SSE notifications are routed here even after localStorage was cleared. const syncServerFolderSessions = useCallback(async () => { - const folders = await smartFolderStorage.getAllFolders(); + const folders = await store.getAllFolders(); for (const folder of folders) { if (!isServerFolderInput(folder)) continue; try { @@ -655,9 +688,9 @@ export function useFolderAutomation(toolRegistry: Partial) { // when the folder was created). Re-provision it now using the stored automation. if (err?.response?.status === 404) { try { - const automation = await automationStorage.getAutomation(folder.automationId); + const automation = await resolveFolderAutomation(folder); if (!automation) { - console.warn(`[watch-folders] Cannot re-provision ${folder.id}: automation ${folder.automationId} not found in IDB`); + console.warn(`[watch-folders] Cannot re-provision ${folder.id}: no automation (id=${folder.automationId}, hasInlineConfig=${!!folder.automationConfig})`); } else { const configJson = buildPipelineJson(automation, toolRegistry); if (!configJson) { @@ -678,29 +711,47 @@ export function useFolderAutomation(toolRegistry: Partial) { } } } - }, [toolRegistry]); + }, [toolRegistry, store]); // ── Lifecycle effects ────────────────────────────────────────────────────── + // Mirror callbacks into refs so the lifecycle effect can mount listeners once + // (SSE / SW / visibility) without tearing them down whenever callback identities + // change. The cost of re-registering the SSE handler is dropping in-flight events. + const storeRef = useRef(store); + const runPipelineRef = useRef(runPipeline); + const drainPendingJobsRef = useRef(drainPendingJobs); + const finalizeFromSSERef = useRef(finalizeFromSSE); + const finalizeFromServerFolderSSERef = useRef(finalizeFromServerFolderSSE); + const finalizeServerFolderErrorRef = useRef(finalizeServerFolderError); + const syncServerFolderSessionsRef = useRef(syncServerFolderSessions); + useEffect(() => { storeRef.current = store; }, [store]); + useEffect(() => { runPipelineRef.current = runPipeline; }, [runPipeline]); + useEffect(() => { drainPendingJobsRef.current = drainPendingJobs; }, [drainPendingJobs]); + useEffect(() => { finalizeFromSSERef.current = finalizeFromSSE; }, [finalizeFromSSE]); + useEffect(() => { finalizeFromServerFolderSSERef.current = finalizeFromServerFolderSSE; }, [finalizeFromServerFolderSSE]); + useEffect(() => { finalizeServerFolderErrorRef.current = finalizeServerFolderError; }, [finalizeServerFolderError]); + useEffect(() => { syncServerFolderSessionsRef.current = syncServerFolderSessions; }, [syncServerFolderSessions]); + useEffect(() => { async function drainDueRetries() { const due = await folderRetryScheduleStorage.claimDue(); for (const entry of due) { - const freshFolder = await smartFolderStorage.getFolder(entry.folderId); + const freshFolder = await storeRef.current.getFolder(entry.folderId); if (!freshFolder || freshFolder.isPaused) continue; const freshFile = await fileStorage.getStirlingFile(entry.fileId as FileId); if (!freshFile) continue; - await folderStorage.updateFileMetadata(entry.folderId, entry.fileId, { + await storeRef.current.updateFileMetadata(entry.folderId, entry.fileId, { status: 'pending', nextRetryAt: undefined, serverJobId: undefined, }); - void runPipeline(freshFolder, freshFile, entry.fileId, entry.ownedByFolder); + void runPipelineRef.current(freshFolder, freshFile, entry.fileId, entry.ownedByFolder); } } void drainDueRetries(); - void drainPendingJobs(); - void syncServerFolderSessions(); + void drainPendingJobsRef.current(); + void syncServerFolderSessionsRef.current(); if ('serviceWorker' in navigator) { navigator.serviceWorker @@ -710,26 +761,27 @@ export function useFolderAutomation(toolRegistry: Partial) { function handleSWMessage(event: MessageEvent) { if (event.data?.type === 'PROCESS_DUE_RETRIES') void drainDueRetries(); - if (event.data?.type === 'POLL_PIPELINE_JOBS') void drainPendingJobs(); + if (event.data?.type === 'POLL_PIPELINE_JOBS') void drainPendingJobsRef.current(); } navigator.serviceWorker?.addEventListener('message', handleSWMessage); function handleVisibilityChange() { if (document.visibilityState === 'visible') { void drainDueRetries(); - void drainPendingJobs(); + void drainPendingJobsRef.current(); } } document.addEventListener('visibilitychange', handleVisibilityChange); - // SSE handler — receives job-complete / job-failed / server-folder-complete / server-folder-error events + // SSE handler — receives job-complete / job-failed / server-folder-complete / server-folder-error events. + // Reads callbacks from refs so identity changes don't force re-subscription. const removeSSEHandler = addSSEHandler((data: unknown) => { const event = parsePipelineSSEEvent(data); if (!event) return; - if (event.type === 'job-complete') void finalizeFromSSE(event.jobId); - if (event.type === 'job-failed') void finalizeFromSSE(event.jobId, event.error ?? 'Server job failed'); - if (event.type === 'server-folder-complete') void finalizeFromServerFolderSSE(event.folderId, event.outputFiles); - if (event.type === 'server-folder-error') void finalizeServerFolderError(event.folderId, event.failedFileIds); + if (event.type === 'job-complete') void finalizeFromSSERef.current(event.jobId); + if (event.type === 'job-failed') void finalizeFromSSERef.current(event.jobId, event.error ?? 'Server job failed'); + if (event.type === 'server-folder-complete') void finalizeFromServerFolderSSERef.current(event.folderId, event.outputFiles); + if (event.type === 'server-folder-error') void finalizeServerFolderErrorRef.current(event.folderId, event.failedFileIds); }); return () => { @@ -737,7 +789,7 @@ export function useFolderAutomation(toolRegistry: Partial) { document.removeEventListener('visibilitychange', handleVisibilityChange); removeSSEHandler(); }; - }, [runPipeline, drainPendingJobs, finalizeFromSSE, finalizeFromServerFolderSSE, finalizeServerFolderError, syncServerFolderSessions]); + }, []); /** Run multiple files through the pipeline concurrently. */ const processBatch = useCallback( diff --git a/frontend/src/core/hooks/useFolderData.ts b/frontend/src/core/hooks/useFolderData.ts index 66dbcd98bd..498f4afcfd 100644 --- a/frontend/src/core/hooks/useFolderData.ts +++ b/frontend/src/core/hooks/useFolderData.ts @@ -5,7 +5,7 @@ import { useState, useEffect, useCallback, useMemo } from "react"; import { FolderFileMetadata, FolderRecord } from "@app/types/smartFolders"; import { folderStorage } from "@app/services/folderStorage"; -import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext"; +import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext"; interface UseFolderDataReturn { folderRecord: FolderRecord | null; @@ -23,23 +23,25 @@ interface UseFolderDataReturn { } export function useFolderData(folderId: string): UseFolderDataReturn { - const backend = useWatchFolderStorage(); + const store = useWatchFolderStore(); const [folderRecord, setFolderRecord] = useState(null); const refresh = useCallback(async () => { if (!folderId) return; try { - const record = backend ? await backend.getFolderData(folderId) : await folderStorage.getFolderData(folderId); + const record = await store.getFolderData(folderId); setFolderRecord(record); } catch (error) { console.error("Failed to load folder data:", error); } - }, [folderId, backend]); + }, [folderId, store]); useEffect(() => { refresh(); }, [refresh]); + // Subscribe to IDB change events — the server backend also writes to IDB, + // so this picks up changes from both local and server-backed operations. useEffect(() => { const unsubscribe = folderStorage.onFolderChange((changedFolderId) => { if (changedFolderId === folderId) { @@ -57,40 +59,28 @@ export function useFolderData(folderId: string): UseFolderDataReturn { const addFile = useCallback( async (fileId: string, metadata?: Partial) => { - if (backend) { - await backend.addFileToFolder(folderId, fileId, metadata); - } else { - await folderStorage.addFileToFolder(folderId, fileId, metadata); - } + await store.addFileToFolder(folderId, fileId, metadata); }, - [folderId, backend], + [folderId, store], ); const removeFile = useCallback( async (fileId: string) => { - await folderStorage.removeFileFromFolder(folderId, fileId); + await store.removeFileFromFolder(folderId, fileId); }, - [folderId], + [folderId, store], ); const updateFileMetadata = useCallback( async (fileId: string, updates: Partial) => { - if (backend) { - await backend.updateFileMetadata(folderId, fileId, updates); - } else { - await folderStorage.updateFileMetadata(folderId, fileId, updates); - } + await store.updateFileMetadata(folderId, fileId, updates); }, - [folderId, backend], + [folderId, store], ); const clearFolder = useCallback(async () => { - if (backend) { - await backend.clearFolder(folderId); - } else { - await folderStorage.clearFolder(folderId); - } - }, [folderId, backend]); + await store.clearFolder(folderId); + }, [folderId, store]); const getFileMetadata = useCallback( (fileId: string): FolderFileMetadata | null => { diff --git a/frontend/src/core/hooks/useFolderMembership.ts b/frontend/src/core/hooks/useFolderMembership.ts index 0560a7e850..a72f30ac23 100644 --- a/frontend/src/core/hooks/useFolderMembership.ts +++ b/frontend/src/core/hooks/useFolderMembership.ts @@ -7,9 +7,11 @@ import { useState, useEffect } from 'react'; import { folderStorage } from '@app/services/folderStorage'; import { useAllSmartFolders } from '@app/hooks/useAllSmartFolders'; +import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext'; export function useFolderMembership(): Map { const folders = useAllSmartFolders(); + const store = useWatchFolderStore(); const [membership, setMembership] = useState>(new Map()); useEffect(() => { @@ -27,7 +29,7 @@ export function useFolderMembership(): Map { }; for (const folder of folders) { try { - const record = await folderStorage.getFolderData(folder.id); + const record = await store.getFolderData(folder.id); if (record) { Object.entries(record.files).forEach(([fileId, meta]) => { add(fileId, folder.id); @@ -45,8 +47,9 @@ export function useFolderMembership(): Map { load(); + // Server backend mirrors writes to IDB so this fires for both backends. return folderStorage.onFolderChange(load); - }, [folders]); + }, [folders, store]); return membership; } diff --git a/frontend/src/core/hooks/useFolderOutputIds.ts b/frontend/src/core/hooks/useFolderOutputIds.ts index 6e17e268e9..d4f5e418e7 100644 --- a/frontend/src/core/hooks/useFolderOutputIds.ts +++ b/frontend/src/core/hooks/useFolderOutputIds.ts @@ -6,9 +6,11 @@ import { useState, useEffect } from 'react'; import { folderStorage } from '@app/services/folderStorage'; import { useAllSmartFolders } from '@app/hooks/useAllSmartFolders'; +import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext'; export function useFolderOutputIds(): Set { const folders = useAllSmartFolders(); + const store = useWatchFolderStore(); const [outputIds, setOutputIds] = useState>(new Set()); useEffect(() => { @@ -18,7 +20,7 @@ export function useFolderOutputIds(): Set { const ids = new Set(); for (const folder of folders) { try { - const record = await folderStorage.getFolderData(folder.id); + const record = await store.getFolderData(folder.id); if (record) { Object.values(record.files).forEach(meta => { const oids = meta?.displayFileIds ?? (meta?.displayFileId ? [meta.displayFileId] : []); @@ -31,8 +33,9 @@ export function useFolderOutputIds(): Set { }; load(); + // Server backend mirrors writes to IDB so this fires for both backends. return folderStorage.onFolderChange(load); - }, [folders]); + }, [folders, store]); return outputIds; } diff --git a/frontend/src/core/hooks/useFolderRunState.ts b/frontend/src/core/hooks/useFolderRunState.ts index ca0b3e9fb6..f3baa158e8 100644 --- a/frontend/src/core/hooks/useFolderRunState.ts +++ b/frontend/src/core/hooks/useFolderRunState.ts @@ -5,50 +5,44 @@ import { useState, useEffect, useCallback } from 'react'; import { SmartFolderRunEntry } from '@app/types/smartFolders'; import { folderRunStateStorage } from '@app/services/folderRunStateStorage'; +import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext'; interface UseFolderRunStateReturn { recentRuns: SmartFolderRunEntry[]; - setRecentRuns: (runs: SmartFolderRunEntry[]) => Promise; clearRecentRuns: () => Promise; isLoading: boolean; } export function useFolderRunState(folderId: string): UseFolderRunStateReturn { + const store = useWatchFolderStore(); const [recentRuns, setRecentRunsState] = useState([]); const [isLoading, setIsLoading] = useState(true); // Subscribe before loading to close the race window between load completing // and the listener being established (writes in that gap would be missed). + // Server backend mirrors writes to IDB so this fires for both backends. useEffect(() => { if (!folderId) return; setIsLoading(true); const unsub = folderRunStateStorage.onRunStateChange((changedFolderId) => { if (changedFolderId !== folderId) return; - folderRunStateStorage + store .getFolderRunState(folderId) .then(setRecentRunsState) .catch((err) => console.error('Failed to reload folder run state:', err)); }); - folderRunStateStorage + store .getFolderRunState(folderId) .then(setRecentRunsState) .catch((err) => console.error('Failed to load folder run state:', err)) .finally(() => setIsLoading(false)); return unsub; - }, [folderId]); - - const setRecentRuns = useCallback( - async (runs: SmartFolderRunEntry[]) => { - await folderRunStateStorage.setFolderRunState(folderId, runs); - setRecentRunsState(runs); - }, - [folderId] - ); + }, [folderId, store]); const clearRecentRuns = useCallback(async () => { - await folderRunStateStorage.clearFolderRunState(folderId); + await store.clearFolderRunState(folderId); setRecentRunsState([]); - }, [folderId]); + }, [folderId, store]); - return { recentRuns, setRecentRuns, clearRecentRuns, isLoading }; + return { recentRuns, clearRecentRuns, isLoading }; } diff --git a/frontend/src/core/hooks/useFolderRunStatuses.ts b/frontend/src/core/hooks/useFolderRunStatuses.ts index e3d04b3b6b..63c1567373 100644 --- a/frontend/src/core/hooks/useFolderRunStatuses.ts +++ b/frontend/src/core/hooks/useFolderRunStatuses.ts @@ -6,7 +6,11 @@ import { useState, useEffect, useRef } from "react"; import { SmartFolder, SmartFolderRunEntry } from "@app/types/smartFolders"; import { folderRunStateStorage } from "@app/services/folderRunStateStorage"; -import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext"; +import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext"; + +// IDB run-state events fire for both backends — the server backend mirrors writes to IDB +// (see serverBackend.addFolderRunEntries), so listening to the IDB event surface gives us +// live updates for both local and server-backed folders. export type FolderRunStatus = "idle" | "processing" | "done"; @@ -23,7 +27,7 @@ function deriveStatus(runs: SmartFolderRunEntry[]): FolderRunStatus { } export function useFolderRunStatuses(folders: SmartFolder[]): Record { - const backend = useWatchFolderStorage(); + const store = useWatchFolderStore(); const [statuses, setStatuses] = useState>({}); const doneTimersRef = useRef>>(new Map()); const foldersRef = useRef(folders); @@ -36,9 +40,7 @@ export function useFolderRunStatuses(folders: SmartFolder[]): Record { try { - const runs = backend - ? await backend.getFolderRunState(folder.id) - : await folderRunStateStorage.getFolderRunState(folder.id); + const runs = await store.getFolderRunState(folder.id); return [folder.id, deriveStatus(runs)] as const; } catch { return [folder.id, "idle" as FolderRunStatus] as const; @@ -53,20 +55,20 @@ export function useFolderRunStatuses(folders: SmartFolder[]): Record { return folderRunStateStorage.onRunStateChange((changedFolderId) => { if (!foldersRef.current.find((f) => f.id === changedFolderId)) return; - folderRunStateStorage + store .getFolderRunState(changedFolderId) .then((runs) => { setStatuses((prev) => ({ ...prev, [changedFolderId]: deriveStatus(runs) })); }) .catch((err) => console.error("Failed to update run status:", err)); }); - }, []); + }, [store]); // When a folder becomes 'done', revert to 'idle' after TTL useEffect(() => { diff --git a/frontend/src/core/hooks/useLocalFolderPoller.ts b/frontend/src/core/hooks/useLocalFolderPoller.ts index 0c115b8ec3..98d068f7ba 100644 --- a/frontend/src/core/hooks/useLocalFolderPoller.ts +++ b/frontend/src/core/hooks/useLocalFolderPoller.ts @@ -8,8 +8,7 @@ import { useEffect, useRef } from 'react'; import { SmartFolder } from '@app/types/smartFolders'; -import { smartFolderStorage } from '@app/services/smartFolderStorage'; -import { folderStorage } from '@app/services/folderStorage'; +import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext'; import { folderDirectoryHandleStorage } from '@app/services/folderDirectoryHandleStorage'; import { folderSeenFilesStorage, makeSeenKey } from '@app/services/folderSeenFilesStorage'; import { resolveInputFile } from '@app/hooks/useFolderAutomation'; @@ -20,8 +19,11 @@ const POLL_INTERVAL_MS = 10_000; export function useLocalFolderPoller( runPipeline: (folder: SmartFolder, file: File, inputFileId: string, ownedByFolder: boolean) => Promise ): void { + const store = useWatchFolderStore(); const runPipelineRef = useRef(runPipeline); + const storeRef = useRef(store); useEffect(() => { runPipelineRef.current = runPipeline; }); + useEffect(() => { storeRef.current = store; }); useEffect(() => { let cancelled = false; @@ -31,7 +33,7 @@ export function useLocalFolderPoller( let folders: SmartFolder[]; try { - folders = await smartFolderStorage.getAllFolders(); + folders = await storeRef.current.getAllFolders(); } catch { return; } @@ -48,7 +50,7 @@ export function useLocalFolderPoller( const hasPermission = await folderDirectoryHandleStorage.ensureReadPermission(inputHandle); if (!hasPermission) continue; - const folderData = await folderStorage.getFolderData(folder.id); + const folderData = await storeRef.current.getFolderData(folder.id); // Build set of file names already in the folder (any status) to avoid duplicates // keyed by name+size (can't use lastModified — file handle gives same value each time) const processingNames = new Set( @@ -84,7 +86,7 @@ export function useLocalFolderPoller( const { inputFileId, ownedByFolder } = await resolveInputFile(file); - await folderStorage.addFileToFolder(folder.id, inputFileId, { + await storeRef.current.addFileToFolder(folder.id, inputFileId, { status: 'pending', name: file.name, ownedByFolder, diff --git a/frontend/src/core/hooks/useSmartFolders.ts b/frontend/src/core/hooks/useSmartFolders.ts index 40e41c0e75..e81fcae3a1 100644 --- a/frontend/src/core/hooks/useSmartFolders.ts +++ b/frontend/src/core/hooks/useSmartFolders.ts @@ -4,15 +4,13 @@ import { useState, useEffect, useCallback } from "react"; import { SmartFolder, isServerFolderInput } from "@app/types/smartFolders"; -import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/services/smartFolderStorage"; -import { folderStorage } from "@app/services/folderStorage"; -import { folderRunStateStorage } from "@app/services/folderRunStateStorage"; + import { fileStorage } from "@app/services/fileStorage"; import { deleteServerFolder } from "@app/services/serverFolderApiService"; import { folderRetryScheduleStorage } from "@app/services/folderRetryScheduleStorage"; import { folderSeenFilesStorage } from "@app/services/folderSeenFilesStorage"; import { folderDirectoryHandleStorage } from "@app/services/folderDirectoryHandleStorage"; -import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext"; +import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext"; import { FileId } from "@app/types/fileContext"; interface UseSmartFoldersReturn { @@ -25,13 +23,12 @@ interface UseSmartFoldersReturn { } export function useSmartFolders(): UseSmartFoldersReturn { - const backend = useWatchFolderStorage(); + const store = useWatchFolderStore(); const [folders, setFolders] = useState([]); const [loading, setLoading] = useState(true); const refreshFolders = useCallback(async () => { try { - const store = backend ?? { getAllFolders: () => smartFolderStorage.getAllFolders() }; const all = await store.getAllFolders(); setFolders(all); } catch (error) { @@ -39,62 +36,45 @@ export function useSmartFolders(): UseSmartFoldersReturn { } finally { setLoading(false); } - }, [backend]); + }, [store]); useEffect(() => { refreshFolders(); }, [refreshFolders]); useEffect(() => { - if (backend) { - return backend.onChange(() => refreshFolders()); - } - const handler = () => { - refreshFolders(); - }; - window.addEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, handler); - return () => window.removeEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, handler); - }, [backend, refreshFolders]); + return store.onChange(() => refreshFolders()); + }, [store, refreshFolders]); const createFolder = useCallback( async (data: Omit): Promise => { - if (backend) return backend.createFolder(data); - return smartFolderStorage.createFolder(data); + return store.createFolder(data); }, - [backend], + [store], ); const updateFolder = useCallback( async (folder: SmartFolder): Promise => { - if (backend) return backend.updateFolder(folder); - return smartFolderStorage.updateFolder(folder); + return store.updateFolder(folder); }, - [backend], + [store], ); const deleteFolder = useCallback( async (id: string): Promise => { - const store = backend ?? { - getFolder: (fid: string) => smartFolderStorage.getFolder(fid), - deleteFolder: (fid: string) => smartFolderStorage.deleteFolder(fid), - getFolderData: (fid: string) => folderStorage.getFolderData(fid), - clearFolder: (fid: string) => folderStorage.clearFolder(fid), - clearFolderRunState: (fid: string) => folderRunStateStorage.clearFolderRunState(fid), - }; - - // Clean up server watch folder first (best-effort — don't block if server is down) + // Clean up server watch folder first (best-effort) const folderMeta = await store.getFolder(id); if (folderMeta && isServerFolderInput(folderMeta)) { await deleteServerFolder(id).catch(() => {}); } - const record = await (backend ? backend.getFolderData(id) : folderStorage.getFolderData(id)); + const record = await store.getFolderData(id); if (record) { - // Only delete input files the folder created from disk — never touch sidebar-sourced files. + // Only delete input files the folder created from disk const ownedInputIds = Object.entries(record.files) .filter(([, meta]) => meta.ownedByFolder === true) .map(([fid]) => fid); - // Always delete every output the folder produced (folder always owns those). + // Always delete every output the folder produced const outputIds = Object.values(record.files).flatMap( (meta) => meta.displayFileIds ?? (meta.displayFileId ? [meta.displayFileId] : []), ); @@ -102,28 +82,18 @@ export function useSmartFolders(): UseSmartFoldersReturn { await Promise.all(toDelete.map((fid) => fileStorage.deleteStirlingFile(fid as FileId).catch(() => {}))); } - if (backend) { - await backend.clearFolder(id); - await backend.clearFolderRunState(id); - } else { - await folderStorage.clearFolder(id); - await folderRunStateStorage.clearFolderRunState(id); - } + await store.clearFolder(id); + await store.clearFolderRunState(id); // These are always IDB-only (browser API objects / ephemeral state) await folderRetryScheduleStorage.clearFolder(id).catch(() => {}); await folderSeenFilesStorage.clearFolder(id).catch(() => {}); await folderDirectoryHandleStorage.remove(id).catch(() => {}); await folderDirectoryHandleStorage.removeInput(id).catch(() => {}); - - if (backend) { - await backend.deleteFolder(id); - } else { - await smartFolderStorage.deleteFolder(id); - } + await store.deleteFolder(id); // Notify the sidebar file list that files have been removed. window.dispatchEvent(new CustomEvent("stirling:files-changed")); }, - [backend], + [store], ); return { folders, loading, createFolder, updateFolder, deleteFolder, refreshFolders }; diff --git a/frontend/src/core/services/folderStorage.ts b/frontend/src/core/services/folderStorage.ts index 7de754c6e9..806ab0a980 100644 --- a/frontend/src/core/services/folderStorage.ts +++ b/frontend/src/core/services/folderStorage.ts @@ -153,15 +153,20 @@ class FolderStorage { }); } - /** Overwrite the entire folder record (used by sync from server). */ - async setFolderData(folderId: string, record: FolderRecord): Promise { + /** + * Overwrite the entire folder record (used by sync from server). + * Pass `{ silent: true }` for server-mirror writes — these reflect a read, + * not a user action, so dispatching change events would cause subscribers + * (which themselves call getFolderData) to re-fetch in an infinite loop. + */ + async setFolderData(folderId: string, record: FolderRecord, opts?: { silent?: boolean }): Promise { const db = await this.ensureDB(); return new Promise((resolve, reject) => { const transaction = db.transaction([this.recordsStore], "readwrite"); const store = transaction.objectStore(this.recordsStore); const request = store.put(record); request.onsuccess = () => { - this.dispatchChange(folderId); + if (!opts?.silent) this.dispatchChange(folderId); resolve(); }; request.onerror = () => reject(new Error("Failed to set folder data")); diff --git a/frontend/src/core/services/watchFolderIdbBackend.ts b/frontend/src/core/services/watchFolderIdbBackend.ts index c3685fb833..a717d94483 100644 --- a/frontend/src/core/services/watchFolderIdbBackend.ts +++ b/frontend/src/core/services/watchFolderIdbBackend.ts @@ -9,7 +9,6 @@ import type { WatchFolderStorageBackend } from "@app/contexts/WatchFolderStorage import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/services/smartFolderStorage"; import { folderStorage } from "@app/services/folderStorage"; import { folderRunStateStorage } from "@app/services/folderRunStateStorage"; -import type { SmartFolder, FolderRecord, FolderFileMetadata, SmartFolderRunEntry } from "@app/types/smartFolders"; export const idbBackend: WatchFolderStorageBackend = { // Folder CRUD @@ -24,6 +23,7 @@ export const idbBackend: WatchFolderStorageBackend = { getFolderData: (folderId) => folderStorage.getFolderData(folderId), updateFileMetadata: (folderId, fileId, meta) => folderStorage.updateFileMetadata(folderId, fileId, meta), addFileToFolder: (folderId, fileId, meta) => folderStorage.addFileToFolder(folderId, fileId, meta), + removeFileFromFolder: (folderId, fileId) => folderStorage.removeFileFromFolder(folderId, fileId), clearFolder: (folderId) => folderStorage.clearFolder(folderId), // Run state diff --git a/frontend/src/core/types/smartFolders.ts b/frontend/src/core/types/smartFolders.ts index 50f257c380..54858fd6c4 100644 --- a/frontend/src/core/types/smartFolders.ts +++ b/frontend/src/core/types/smartFolders.ts @@ -14,16 +14,16 @@ export interface SmartFolder { order?: number; isDefault?: boolean; isPaused?: boolean; - maxRetries?: number; // 0 = disabled; default 3 + maxRetries?: number; // 0 = disabled; default 3 retryDelayMinutes?: number; // default 5 - outputMode?: 'new_file' | 'new_version'; // default: 'new_file' (existing behaviour) - outputName?: string; // output filename prefix/suffix - outputNamePosition?: 'prefix' | 'suffix' | 'auto-number'; // default: 'prefix' - hasOutputDirectory?: boolean; // true when a local FS output folder is configured + outputMode?: "new_file" | "new_version"; // default: 'new_file' (existing behaviour) + outputName?: string; // output filename prefix/suffix + outputNamePosition?: "prefix" | "suffix" | "auto-number"; // default: 'prefix' + hasOutputDirectory?: boolean; // true when a local FS output folder is configured /** Where input files come from. Default: 'idb' (dropped/sidebar files stay in browser). */ - inputSource?: 'idb' | 'local-folder' | 'server-folder'; + inputSource?: "idb" | "local-folder" | "server-folder"; /** Where processing happens. Default: 'local' (browser). Forced to 'server' when inputSource='server-folder'. */ - processingMode?: 'local' | 'server'; + processingMode?: "local" | "server"; /** * How long to keep output files in the server's processed/ dir (hours). * null / undefined = keep forever. Only meaningful when inputSource='server-folder'. @@ -34,11 +34,18 @@ export interface SmartFolder { * immediately after downloading it. Only meaningful when inputSource='server-folder'. */ deleteOutputOnDownload?: boolean; + /** Visibility: PERSONAL (owner-only) or ORGANISATION (all users). Only set when server-backed. */ + scope?: "PERSONAL" | "ORGANISATION"; + /** + * Inlined automation pipeline (JSON-stringified operations array). + * Used by the server DB instead of automationId. When present, takes precedence. + */ + automationConfig?: string; } export interface FolderFileMetadata { addedAt: Date; - status: 'pending' | 'processing' | 'processed' | 'error'; + status: "pending" | "processing" | "processed" | "error"; processedAt?: Date; /** All output file ids produced by this run — references stirling-pdf-files */ displayFileIds?: string[]; @@ -64,7 +71,7 @@ export interface FolderFileMetadata { /** Type guard / helper — true when the folder's input source is the server watch folder. */ export function isServerFolderInput(folder: SmartFolder): boolean { - return folder.inputSource === 'server-folder'; + return folder.inputSource === "server-folder"; } export interface FolderRecord { @@ -81,5 +88,5 @@ export interface SmartFolderRunEntry { displayFileIds?: string[]; /** When this run completed — used for TTL-based "done" status */ processedAt?: Date; - status: 'processing' | 'processed'; + status: "processing" | "processed"; } diff --git a/frontend/src/proprietary/components/WatchFolderServerProvider.tsx b/frontend/src/proprietary/components/WatchFolderServerProvider.tsx index ad87ec5006..587ce7e600 100644 --- a/frontend/src/proprietary/components/WatchFolderServerProvider.tsx +++ b/frontend/src/proprietary/components/WatchFolderServerProvider.tsx @@ -1,22 +1,114 @@ /** * Proprietary wrapper that overrides the core IDB-only WatchFolderStorageProvider * with the server-backed implementation when premium is enabled. + * + * On first premium load, migrates existing IDB folders, file metadata, and run + * history to the server. The flag is only set when ALL upserts succeed; partial + * failures cause a retry on the next load. */ -import React from "react"; +import React, { useEffect, useRef } from "react"; import { useAppConfig } from "@app/contexts/AppConfigContext"; import { WatchFolderStorageProvider } from "@app/contexts/WatchFolderStorageContext"; import { serverBackend } from "@proprietary/services/watchFolderServerBackend"; +import { smartFolderStorage } from "@app/services/smartFolderStorage"; +import { folderStorage } from "@app/services/folderStorage"; +import { folderRunStateStorage } from "@app/services/folderRunStateStorage"; +import { watchFolderApi } from "@proprietary/services/watchFolderApiService"; + +const MIGRATION_KEY = "watch_folders_migrated_to_server"; +const MIGRATION_LOCK = "watch-folders-migration"; + +async function migrateOne(folderId: string, serverIds: Set): Promise { + const folder = await smartFolderStorage.getFolder(folderId); + if (!folder) return true; + + // 1. Folder definition (skip if already on server — another tab/device may have done it) + if (!serverIds.has(folder.id)) { + await serverBackend.createFolderWithId(folder); + } + + // 2. File metadata + const record = await folderStorage.getFolderData(folder.id); + if (record) { + for (const [fileId, meta] of Object.entries(record.files)) { + await serverBackend.addFileToFolder(folder.id, fileId, meta); + // updateFileMetadata pushes the full snapshot; addFileToFolder only pushes the basics. + await serverBackend.updateFileMetadata(folder.id, fileId, meta); + } + } + + // 3. Run history + const runs = await folderRunStateStorage.getFolderRunState(folder.id); + if (runs.length > 0) { + await serverBackend.addFolderRunEntries(folder.id, runs); + } + + return true; +} + +async function runMigration(): Promise { + if (localStorage.getItem(MIGRATION_KEY)) return true; + + const idbFolders = await smartFolderStorage.getAllFolders(); + if (idbFolders.length === 0) { + localStorage.setItem(MIGRATION_KEY, "1"); + return true; + } + + const serverFolders = await watchFolderApi.list(); + const serverIds = new Set(serverFolders.map((f) => f.id)); + + let allOk = true; + for (const folder of idbFolders) { + try { + await migrateOne(folder.id, serverIds); + } catch (err) { + console.warn(`[watch-folders] Migration of folder ${folder.id} failed:`, err); + allOk = false; + } + } + + // Only commit the flag when every folder migrated cleanly; otherwise retry next load. + if (allOk) localStorage.setItem(MIGRATION_KEY, "1"); + return allOk; +} export function WatchFolderServerProvider({ children }: { children: React.ReactNode }) { const { config } = useAppConfig(); const isPremium = config?.premiumEnabled === true; + const migrationRan = useRef(false); + + useEffect(() => { + if (!isPremium || migrationRan.current) return; + if (localStorage.getItem(MIGRATION_KEY)) return; + migrationRan.current = true; + + // Use Web Locks API when available so concurrent tabs don't double-migrate; + // fall back to a plain run when the API is missing (older browsers / SSR). + const run = async () => { + try { + await runMigration(); + } catch (err) { + console.warn("[watch-folders] Migration aborted:", err); + } + }; + + if ("locks" in navigator) { + navigator.locks.request(MIGRATION_LOCK, { ifAvailable: true }, async (lock) => { + // ifAvailable: lock is null if another tab has it — skip; that tab will commit the flag. + if (!lock) return; + if (localStorage.getItem(MIGRATION_KEY)) return; // re-check after lock acquired + await run(); + }); + } else { + void run(); + } + }, [isPremium]); if (!isPremium) { - // Core's IDB provider is already in place — just pass through return <>{children}; } - // Override with server-backed storage return {children}; } diff --git a/frontend/src/proprietary/services/watchFolderApiService.ts b/frontend/src/proprietary/services/watchFolderApiService.ts index 64d2becd87..d7a8598bae 100644 --- a/frontend/src/proprietary/services/watchFolderApiService.ts +++ b/frontend/src/proprietary/services/watchFolderApiService.ts @@ -102,6 +102,10 @@ export const watchFolderApi = { await apiClient.delete(`${BASE}/${folderId}/files`); }, + async deleteFile(folderId: string, fileId: string): Promise { + await apiClient.delete(`${BASE}/${folderId}/files/${encodeURIComponent(fileId)}`); + }, + // Runs async listRuns(folderId: string): Promise { const res = await apiClient.get(`${BASE}/${folderId}/runs`); @@ -117,4 +121,8 @@ export const watchFolderApi = { const res = await apiClient.post(`${BASE}/${folderId}/runs/batch`, runs); return res.data; }, + + async deleteRuns(folderId: string): Promise { + await apiClient.delete(`${BASE}/${folderId}/runs`); + }, }; diff --git a/frontend/src/proprietary/services/watchFolderServerBackend.ts b/frontend/src/proprietary/services/watchFolderServerBackend.ts index c11ee7c428..9b873ca920 100644 --- a/frontend/src/proprietary/services/watchFolderServerBackend.ts +++ b/frontend/src/proprietary/services/watchFolderServerBackend.ts @@ -3,7 +3,7 @@ * * Writes go to the server API first (source of truth), then mirror to IDB for fast reads. * Reads come from IDB (populated on init and after writes). - * Falls back to IDB-only if the server is unreachable. + * Falls back to IDB-only on network errors (NOT on auth/permission errors). */ import type { WatchFolderStorageBackend } from "@app/contexts/WatchFolderStorageContext"; @@ -11,16 +11,38 @@ import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/serv import { folderStorage } from "@app/services/folderStorage"; import { folderRunStateStorage } from "@app/services/folderRunStateStorage"; import type { SmartFolder, FolderRecord, FolderFileMetadata, SmartFolderRunEntry } from "@app/types/smartFolders"; -import { watchFolderApi, WatchFolderDTO, WatchFolderFileDTO, WatchFolderRunDTO } from "./watchFolderApiService"; +import { watchFolderApi, WatchFolderDTO } from "./watchFolderApiService"; +import { AxiosError } from "axios"; -// ── DTO ↔ Domain conversions ─────────────────────────────────────────────── +// ── Error classification ────────────────────────────────────────────────── + +/** Returns true only for network/timeout errors where fallback to IDB makes sense. */ +function isNetworkError(err: unknown): boolean { + if (err instanceof AxiosError) { + // No response at all = network failure / timeout + if (!err.response) return true; + // 5xx = server error, safe to fall back + if (err.response.status >= 500) return true; + // 401/403 = auth issue — don't silently fall back to stale IDB data + return false; + } + return true; // Unknown error type — treat as network +} + +/** True if the error is a 404 — treat as "not found" rather than a fatal error. */ +function isNotFound(err: unknown): boolean { + return err instanceof AxiosError && err.response?.status === 404; +} + +// ── DTO ↔ Domain conversions ────────────────────────────────────────────── function toSmartFolder(dto: WatchFolderDTO): SmartFolder { return { id: dto.id, name: dto.name, description: dto.description ?? "", - automationId: "", // automation config is inlined — hooks resolve this + automationId: "", // server uses inlined automationConfig instead + automationConfig: dto.automationConfig, icon: dto.icon ?? "FolderIcon", accentColor: dto.accentColor ?? "#3b82f6", createdAt: dto.createdAt ?? new Date().toISOString(), @@ -28,6 +50,7 @@ function toSmartFolder(dto: WatchFolderDTO): SmartFolder { order: dto.orderIndex, isDefault: dto.isDefault, isPaused: dto.isPaused, + scope: dto.scope, inputSource: (dto.inputSource as SmartFolder["inputSource"]) ?? "idb", processingMode: (dto.processingMode as SmartFolder["processingMode"]) ?? "local", outputMode: (dto.outputMode as SmartFolder["outputMode"]) ?? "new_file", @@ -40,14 +63,15 @@ function toSmartFolder(dto: WatchFolderDTO): SmartFolder { }; } -function toDTO(folder: SmartFolder & { scope?: string }): WatchFolderDTO { +function toDTO(folder: SmartFolder): WatchFolderDTO { return { id: folder.id, name: folder.name, description: folder.description, + automationConfig: folder.automationConfig, icon: folder.icon, accentColor: folder.accentColor, - scope: (folder as any).scope ?? "PERSONAL", + scope: folder.scope ?? "PERSONAL", orderIndex: folder.order, isDefault: folder.isDefault, isPaused: folder.isPaused, @@ -67,11 +91,22 @@ function dispatchChange() { window.dispatchEvent(new Event(SMART_FOLDER_STORAGE_CHANGE_EVENT)); } -// ── Sync helper: pull server state into IDB ──────────────────────────────── +// ── Sync helper: pull server state into IDB ─────────────────────────────── + +let lastSyncAt = 0; +const SYNC_DEBOUNCE_MS = 10_000; // Don't re-sync more than once per 10s + +async function syncFoldersToIdb(force = false): Promise { + const now = Date.now(); + if (!force && now - lastSyncAt < SYNC_DEBOUNCE_MS) { + // Return IDB cache — it's fresh enough + return smartFolderStorage.getAllFolders(); + } -async function syncFoldersToIdb(): Promise { const dtos = await watchFolderApi.list(); const folders = dtos.map(toSmartFolder); + lastSyncAt = Date.now(); + // Overwrite IDB with server state const existing = await smartFolderStorage.getAllFolders(); const serverIds = new Set(folders.map((f) => f.id)); @@ -89,14 +124,14 @@ async function syncFoldersToIdb(): Promise { return folders; } -// ── Backend implementation ───────────────────────────────────────────────── +// ── Backend implementation ──────────────────────────────────────────────── export const serverBackend: WatchFolderStorageBackend = { async getAllFolders() { try { return await syncFoldersToIdb(); - } catch { - // Server unreachable — fall back to IDB + } catch (err) { + if (!isNetworkError(err)) throw err; return smartFolderStorage.getAllFolders(); } }, @@ -107,7 +142,11 @@ export const serverBackend: WatchFolderStorageBackend = { const folder = toSmartFolder(dto); await smartFolderStorage.createFolderWithId(folder).catch(() => {}); return folder; - } catch { + } catch (err) { + // 404 → folder genuinely doesn't exist on server (or was deleted in another tab). + // Don't fall back to IDB; return null so callers can treat as gone. + if (isNotFound(err)) return null; + if (!isNetworkError(err)) throw err; return smartFolderStorage.getFolder(id); } }, @@ -124,10 +163,11 @@ export const serverBackend: WatchFolderStorageBackend = { const created = await watchFolderApi.create(toDTO(folder)); const result = toSmartFolder(created); await smartFolderStorage.createFolderWithId(result).catch(() => {}); + lastSyncAt = 0; // Invalidate sync cache dispatchChange(); return result; - } catch { - // Offline — create in IDB only + } catch (err) { + if (!isNetworkError(err)) throw err; return smartFolderStorage.createFolder(data); } }, @@ -137,9 +177,11 @@ export const serverBackend: WatchFolderStorageBackend = { const created = await watchFolderApi.create(toDTO(folder)); const result = toSmartFolder(created); await smartFolderStorage.createFolderWithId(result).catch(() => {}); + lastSyncAt = 0; dispatchChange(); return result; - } catch { + } catch (err) { + if (!isNetworkError(err)) throw err; return smartFolderStorage.createFolderWithId(folder); } }, @@ -149,9 +191,11 @@ export const serverBackend: WatchFolderStorageBackend = { const updated = await watchFolderApi.update(folder.id, toDTO(folder)); const result = toSmartFolder(updated); await smartFolderStorage.createFolderWithId(result).catch(() => {}); + lastSyncAt = 0; dispatchChange(); return result; - } catch { + } catch (err) { + if (!isNetworkError(err)) throw err; return smartFolderStorage.updateFolder(folder); } }, @@ -159,8 +203,14 @@ export const serverBackend: WatchFolderStorageBackend = { async deleteFolder(id) { try { await watchFolderApi.remove(id); - } catch { - // best-effort server delete + lastSyncAt = 0; + } catch (err) { + // 404 = already deleted — that's success for delete semantics. + if (isNotFound(err)) { + lastSyncAt = 0; + } else if (!isNetworkError(err)) { + throw err; + } } await smartFolderStorage.deleteFolder(id); }, @@ -188,10 +238,13 @@ export const serverBackend: WatchFolderStorageBackend = { processedAt: f.processedAt ? new Date(f.processedAt) : undefined, }; } - // Mirror to IDB - await folderStorage.setFolderData(folderId, record).catch(() => {}); + // Mirror to IDB silently — this is a cache write reflecting a read, not a user action. + // Dispatching FOLDER_CHANGE_EVENT here would cause subscribers (useFolderData, etc.) + // to re-call getFolderData → server → mirror → … infinite loop. + await folderStorage.setFolderData(folderId, record, { silent: true }).catch(() => {}); return record; - } catch { + } catch (err) { + if (!isNetworkError(err)) throw err; return folderStorage.getFolderData(folderId); } }, @@ -199,7 +252,7 @@ export const serverBackend: WatchFolderStorageBackend = { async updateFileMetadata(folderId, fileId, meta) { // Update IDB immediately for fast UI await folderStorage.updateFileMetadata(folderId, fileId, meta); - // Sync to server + // Sync to server — only swallow network failures; surface auth/config errors. try { const existing = await folderStorage.getFolderData(folderId); const fileMeta = existing?.files[fileId]; @@ -218,8 +271,8 @@ export const serverBackend: WatchFolderStorageBackend = { processedAt: fileMeta.processedAt?.toISOString(), }); } - } catch { - // Server sync failed — IDB is still up to date + } catch (err) { + if (!isNetworkError(err)) throw err; } }, @@ -233,16 +286,33 @@ export const serverBackend: WatchFolderStorageBackend = { ownedByFolder: meta?.ownedByFolder, addedAt: meta?.addedAt?.toISOString() ?? new Date().toISOString(), }); - } catch { - // offline — IDB has the data + } catch (err) { + if (!isNetworkError(err)) throw err; + } + }, + + async removeFileFromFolder(folderId, fileId) { + await folderStorage.removeFileFromFolder(folderId, fileId); + try { + await watchFolderApi.deleteFile(folderId, fileId); + } catch (err) { + if (isNotFound(err)) { + // Already gone server-side — DELETE is idempotent. + } else if (!isNetworkError(err)) { + throw err; + } } }, async clearFolder(folderId) { try { await watchFolderApi.deleteFiles(folderId); - } catch { - /* best-effort */ + } catch (err) { + if (isNotFound(err)) { + // Already cleared on server — proceed. + } else if (!isNetworkError(err)) { + throw err; + } } await folderStorage.clearFolder(folderId); }, @@ -259,7 +329,8 @@ export const serverBackend: WatchFolderStorageBackend = { processedAt: r.processedAt ? new Date(r.processedAt) : undefined, })); return entries; - } catch { + } catch (err) { + if (!isNetworkError(err)) throw err; return folderRunStateStorage.getFolderRunState(folderId); } }, @@ -279,18 +350,22 @@ export const serverBackend: WatchFolderStorageBackend = { processedAt: e.processedAt?.toISOString(), })), ); - } catch { - /* offline */ + } catch (err) { + if (!isNetworkError(err)) throw err; } }, async clearFolderRunState(folderId) { - try { - // No dedicated endpoint yet — runs are deleted with the folder - } catch { - /* best-effort */ - } await folderRunStateStorage.clearFolderRunState(folderId); + try { + await watchFolderApi.deleteRuns(folderId); + } catch (err) { + if (isNotFound(err)) { + // Folder already gone or no runs — fine. + } else if (!isNetworkError(err)) { + throw err; + } + } }, onChange(callback) {