haeaeae/Stirling-PDF
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved database

Browse Source
This commit is contained in:
Reece
2026-04-30 22:23:00 +01:00
parent 74b222d957
commit 0f7dbc0993
30 changed files with 540 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/proprietary/src/main/java/stirling/software/proprietary/controller/api/WatchFolderController.java
+15
View File
@@ -78,6 +78,15 @@ public class WatchFolderController {
return ResponseEntity.noContent().build();
}
@DeleteMapping("/{folderId}/files/{fileId}")
public ResponseEntity<Void> deleteFile(
@PathVariable String folderId, @PathVariable String fileId) {
service.deleteFile(folderId, fileId);
// Return 204 whether the row existed or not — DELETE is idempotent and the caller's
// intent ("ensure this row is gone") is satisfied either way.
return ResponseEntity.noContent().build();
}
// ── Folder runs ────────────────────────────────────────────────────────
@GetMapping("/{folderId}/runs")
@@ -97,4 +106,10 @@ public class WatchFolderController {
@Valid @RequestBody @Size(max = RUNS_BATCH_MAX) List<WatchFolderRun> runs) {
return ResponseEntity.status(HttpStatus.CREATED).body(service.addRuns(folderId, runs));
}
@DeleteMapping("/{folderId}/runs")
public ResponseEntity<Void> deleteRuns(@PathVariable String folderId) {
service.deleteRuns(folderId);
return ResponseEntity.noContent().build();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/model/WatchFolder.java
+4 -4
View File
@@ -41,10 +41,10 @@ public class WatchFolder implements Serializable {
/**
* Client-supplied identifier (matches the IndexedDB folder id used by the React frontend).
* Using a String PK — rather than an IDENTITY {@code Long} like other entities — keeps the
* same opaque id across the local IDB cache and the server, so the frontend can round-trip a
* folder between offline/local mode and server mode without remapping. Callers are expected
* to supply a UUID (or equivalently collision-resistant string).
* Using a String PK — rather than an IDENTITY {@code Long} like other entities — keeps the same
* opaque id across the local IDB cache and the server, so the frontend can round-trip a folder
* between offline/local mode and server mode without remapping. Callers are expected to supply
* a UUID (or equivalently collision-resistant string).
*/
@Id
@EqualsAndHashCode.Include
app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/FolderScope.java
+2 -2
View File
@@ -10,9 +10,9 @@ import jakarta.persistence.Converter;
* Visibility scope of a {@code WatchFolder}.
*
* <ul>
* <li>{@link #PERSONAL} — only the owner can see / modify the folder.</li>
* <li>{@link #PERSONAL} — only the owner can see / modify the folder.
* <li>{@link #ORGANISATION} — visible to every authenticated user; only admins may create or
* modify.</li>
* modify.
* </ul>
*/
public enum FolderScope {
app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/InputSource.java
+3 -3
View File
@@ -10,9 +10,9 @@ import jakarta.persistence.Converter;
* Where input files for a watch folder come from.
*
* <ul>
* <li>{@link #IDB} — files dropped / picked in the browser, stored in IndexedDB.</li>
* <li>{@link #LOCAL_FOLDER} — a real folder on the user's machine (desktop build).</li>
* <li>{@link #SERVER_FOLDER} — a directory watched on the server.</li>
* <li>{@link #IDB} — files dropped / picked in the browser, stored in IndexedDB.
* <li>{@link #LOCAL_FOLDER} — a real folder on the user's machine (desktop build).
* <li>{@link #SERVER_FOLDER} — a directory watched on the server.
* </ul>
*/
public enum InputSource {
app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/OutputMode.java
+2 -2
View File
@@ -10,9 +10,9 @@ import jakarta.persistence.Converter;
* How automation output files are produced.
*
* <ul>
* <li>{@link #NEW_FILE} — always produce a new, separately-named file.</li>
* <li>{@link #NEW_FILE} — always produce a new, separately-named file.
* <li>{@link #NEW_VERSION} — produce a new version of the input file (replacing / versioning
* semantics handled client-side).</li>
* semantics handled client-side).
* </ul>
*/
public enum OutputMode {
app/proprietary/src/main/java/stirling/software/proprietary/model/watchfolder/ProcessingMode.java
+2 -2
View File
@@ -10,9 +10,9 @@ import jakarta.persistence.Converter;
* Where the automation pipeline runs.
*
* <ul>
* <li>{@link #LOCAL} — runs entirely in the user's browser.</li>
* <li>{@link #LOCAL} — runs entirely in the user's browser.
* <li>{@link #SERVER} — runs on the server (forced when {@link InputSource#SERVER_FOLDER} is in
* use).</li>
* use).
* </ul>
*/
public enum ProcessingMode {
app/proprietary/src/main/java/stirling/software/proprietary/repository/WatchFolderFileRepository.java
+6
View File
@@ -23,4 +23,10 @@ public interface WatchFolderFileRepository extends JpaRepository<WatchFolderFile
@Transactional
@Query("DELETE FROM WatchFolderFile f WHERE f.folder.id = :folderId")
int deleteAllByFolderId(@Param("folderId") String folderId);
@Modifying
@Transactional
@Query("DELETE FROM WatchFolderFile f WHERE f.folder.id = :folderId AND f.fileId = :fileId")
int deleteByFolderIdAndFileId(
@Param("folderId") String folderId, @Param("fileId") String fileId);
}
app/proprietary/src/main/java/stirling/software/proprietary/service/WatchFolderService.java
+19 -4
View File
@@ -77,8 +77,7 @@ public class WatchFolderService {
WatchFolder existing =
folderRepo
.findById(id)
.orElseThrow(
() -> new IllegalArgumentException("Folder not found: " + id));
.orElseThrow(() -> new IllegalArgumentException("Folder not found: " + id));
requireWriteAccess(existing);
existing.setName(updates.getName());
@@ -114,8 +113,7 @@ public class WatchFolderService {
WatchFolder folder =
folderRepo
.findById(id)
.orElseThrow(
() -> new IllegalArgumentException("Folder not found: " + id));
.orElseThrow(() -> new IllegalArgumentException("Folder not found: " + id));
requireWriteAccess(folder);
// Don't rely on CascadeType.ALL to remove children one-row-at-a-time — for a folder with
@@ -196,6 +194,16 @@ public class WatchFolderService {
fileRepo.deleteAllByFolderId(folderId);
}
/**
* Remove a single file row from a folder. Returns true if a row was deleted, false if no
* matching row existed (already gone, or never present). Idempotent — calling twice is safe.
*/
@Transactional
public boolean deleteFile(String folderId, String fileId) {
requireWriteAccess(folderId);
return fileRepo.deleteByFolderIdAndFileId(folderId, fileId) > 0;
}
// ── Folder runs ────────────────────────────────────────────────────────
@Transactional(readOnly = true)
@@ -229,6 +237,13 @@ public class WatchFolderService {
return runRepo.saveAll(runs);
}
/** Bulk-delete all runs for a folder. */
@Transactional
public void deleteRuns(String folderId) {
requireWriteAccess(folderId);
runRepo.deleteAllByFolderId(folderId);
}
// ── Auth helpers ───────────────────────────────────────────────────────
private User currentUser() {
frontend/src/core/components/smartFolders/SmartFolderHomePage.tsx
+15 -12
View File
@@ -12,13 +12,13 @@ import PauseCircleOutlineIcon from "@mui/icons-material/PauseCircleOutline";
import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline";
import { useSmartFolders } from "@app/hooks/useSmartFolders";
import { useFolderRunStatuses } from "@app/hooks/useFolderRunStatuses";
import { useFolderAutomation, resolveInputFile } from "@app/hooks/useFolderAutomation";
import { useFolderAutomation, resolveInputFile, resolveFolderAutomation } from "@app/hooks/useFolderAutomation";
import { SmartFolder } from "@app/types/smartFolders";
import { AutomationConfig } from "@app/types/automation";
import { iconMap } from "@app/components/tools/automate/iconMap";
import { automationStorage } from "@app/services/automationStorage";
import { folderStorage } from "@app/services/folderStorage";
import { fileStorage } from "@app/services/fileStorage";
import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext";
import { SmartFolderManagementModal } from "@app/components/smartFolders/SmartFolderManagementModal";
import { DeleteFolderConfirmModal } from "@app/components/smartFolders/DeleteFolderConfirmModal";
import { useToolWorkflow } from "@app/contexts/ToolWorkflowContext";
@@ -66,6 +66,7 @@ function FolderCard({
onTogglePause,
}: FolderCardProps) {
const { t } = useTranslation();
const store = useWatchFolderStore();
const [automation, setAutomation] = useState<AutomationConfig | null>(null);
const [fileCount, setFileCount] = useState(0);
const [lastAdded, setLastAdded] = useState<Date | null>(null);
@@ -73,10 +74,10 @@ function FolderCard({
const [isHovered, setIsHovered] = useState(false);
useEffect(() => {
automationStorage.getAutomation(folder.automationId).then(setAutomation);
resolveFolderAutomation(folder).then(setAutomation);
const loadData = () =>
folderStorage.getFolderData(folder.id).then((record) => {
store.getFolderData(folder.id).then((record) => {
if (!record) {
setFileCount(0);
setLastAdded(null);
@@ -89,11 +90,12 @@ function FolderCard({
});
loadData();
// Server backend mirrors writes to IDB so this fires for both backends.
const unsub = folderStorage.onFolderChange((changedId) => {
if (changedId === folder.id) loadData();
});
return unsub;
}, [folder.id, folder.automationId]);
}, [folder.id, folder.automationId, folder.automationConfig, store]);
const FolderIcon = iconMap[folder.icon as keyof typeof iconMap] ?? iconMap.FolderIcon;
const isPaused = folder.isPaused ?? false;
@@ -488,6 +490,7 @@ export function SmartFolderHomePage() {
const { toolRegistry, setCustomWorkbenchViewData } = useToolWorkflow();
const { actions } = useNavigationActions();
const { processBatch } = useFolderAutomation(toolRegistry);
const store = useWatchFolderStore();
const [createModalOpen, setCreateModalOpen] = useState(false);
const [editFolder, setEditFolder] = useState<SmartFolder | null>(null);
@@ -505,7 +508,7 @@ export function SmartFolderHomePage() {
const handleEdit = useCallback(async (folder: SmartFolder) => {
setEditFolder(folder);
const automation = await automationStorage.getAutomation(folder.automationId);
const automation = await resolveFolderAutomation(folder);
setEditAutomation(automation);
setCreateModalOpen(true);
}, []);
@@ -522,7 +525,7 @@ export function SmartFolderHomePage() {
if (pdfs.length === 0) return;
// Load existing folder data once to detect re-runs (same sidebar file dropped again)
const existingData = await folderStorage.getFolderData(folder.id);
const existingData = await store.getFolderData(folder.id);
// Register files sequentially — addFileToFolder/updateFileMetadata are read-modify-write
// without IDB transactions, so concurrent calls on the same folder lose updates.
const items: Array<{ file: File; inputFileId: string; ownedByFolder: boolean }> = [];
@@ -530,9 +533,9 @@ export function SmartFolderHomePage() {
const { inputFileId, ownedByFolder } = await resolveInputFile(file);
if (existingData?.files[inputFileId]) {
// Re-processing: preserve addedAt and ownedByFolder, just reset status
await folderStorage.updateFileMetadata(folder.id, inputFileId, { status: "pending", errorMessage: undefined });
await store.updateFileMetadata(folder.id, inputFileId, { status: "pending", errorMessage: undefined });
} else {
await folderStorage.addFileToFolder(folder.id, inputFileId, {
await store.addFileToFolder(folder.id, inputFileId, {
status: "pending",
name: file.name,
ownedByFolder: ownedByFolder || undefined,
@@ -557,7 +560,7 @@ export function SmartFolderHomePage() {
});
}
},
[processBatch],
[processBatch, store],
);
const handleTogglePause = useCallback(
@@ -569,7 +572,7 @@ export function SmartFolderHomePage() {
if (resuming) {
// Process any files that were queued while paused
const record = await folderStorage.getFolderData(folder.id);
const record = await store.getFolderData(folder.id);
if (record) {
const pendingEntries = Object.entries(record.files).filter(([, meta]) => meta.status === "pending");
if (pendingEntries.length > 0) {
@@ -585,7 +588,7 @@ export function SmartFolderHomePage() {
}
}
},
[updateFolder, refreshFolders, processFiles],
[updateFolder, refreshFolders, processBatch, store],
);
const handleDropSidebarFile = useCallback(
frontend/src/core/components/smartFolders/SmartFolderManagementModal.tsx
+13 -4
View File
@@ -21,7 +21,7 @@ import { AutomationConfig, AutomationMode } from '@app/types/automation';
import { IconPicker as IconSelector } from '@app/components/smartFolders/IconPicker';
import AutomationCreation from '@app/components/tools/automate/AutomationCreation';
import { useToolWorkflow } from '@app/contexts/ToolWorkflowContext';
import { smartFolderStorage } from '@app/services/smartFolderStorage';
import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext';
import { folderDirectoryHandleStorage } from '@app/services/folderDirectoryHandleStorage';
import {
createServerFolder,
@@ -75,6 +75,7 @@ export function SmartFolderManagementModal({
}: SmartFolderManagementModalProps) {
const { t } = useTranslation();
const { toolRegistry } = useToolWorkflow();
const store = useWatchFolderStore();
const isEditMode = !!editFolder;
// Animation state
@@ -186,12 +187,20 @@ export function SmartFolderManagementModal({
const retryFields = { maxRetries, retryDelayMinutes };
const hasOutputDirectory = outputDirName !== null;
const ttlHoursNum = isServerFolder && outputTtlHours !== 'forever' ? Number(outputTtlHours) : null;
// Inline the automation config when persisting to a server-backed store —
// server folders use this instead of looking up automationId in browser IDB.
const inlinedAutomationConfig = JSON.stringify({
name: automation.name,
description: automation.description,
operations: automation.operations,
});
const folderData = {
name: trimmedName,
description: '',
icon,
accentColor,
automationId: automation.id,
automationConfig: inlinedAutomationConfig,
...retryFields,
outputMode: outputMode === 'new_version' ? 'new_version' as const : undefined,
outputName: outputName.trim() || undefined,
@@ -206,7 +215,7 @@ export function SmartFolderManagementModal({
if (isEditMode && editFolder) {
const wasServerFolder = editFolder.inputSource === 'server-folder';
const wasLocalFolder = editFolder.inputSource === 'local-folder';
await smartFolderStorage.updateFolder({ ...editFolder, ...folderData });
await store.updateFolder({ ...editFolder, ...folderData });
if (pendingDirHandle.current) {
await folderDirectoryHandleStorage.set(editFolder.id, pendingDirHandle.current);
} else if (!hasOutputDirectory) {
@@ -229,7 +238,7 @@ export function SmartFolderManagementModal({
await deleteServerFolder(editFolder.id).catch(() => {});
}
} else {
const newFolder = await smartFolderStorage.createFolder(folderData);
const newFolder = await store.createFolder(folderData);
if (pendingDirHandle.current) {
await folderDirectoryHandleStorage.set(newFolder.id, pendingDirHandle.current);
}
@@ -249,7 +258,7 @@ export function SmartFolderManagementModal({
} finally {
setSaving(false);
}
}, [name, icon, accentColor, outputMode, outputName, outputNamePosition, outputDirName, maxRetries, retryDelayMinutes, inputSource, outputTtlHours, deleteOutputOnDownload, isEditMode, editFolder, toolRegistry, resetState, onSaved, onClose, t]);
}, [name, icon, accentColor, outputMode, outputName, outputNamePosition, outputDirName, maxRetries, retryDelayMinutes, inputSource, outputTtlHours, deleteOutputOnDownload, isEditMode, editFolder, toolRegistry, resetState, onSaved, onClose, t, store]);
const handleSave = () => {
const trimmedName = name.trim();
frontend/src/core/components/smartFolders/SmartFolderSection.tsx
+2 -2
View File
@@ -11,7 +11,7 @@ import { DeleteFolderConfirmModal } from '@app/components/smartFolders/DeleteFol
import { SmartFolderCard } from '@app/components/smartFolders/SmartFolderCard';
import { SmartFolder } from '@app/types/smartFolders';
import { AutomationConfig } from '@app/types/automation';
import { automationStorage } from '@app/services/automationStorage';
import { resolveFolderAutomation } from '@app/hooks/useFolderAutomation';
import { SMART_FOLDER_VIEW_ID, SMART_FOLDER_WORKBENCH_ID } from '@app/components/smartFolders/SmartFoldersRegistration';
export function SmartFolderSection() {
@@ -37,7 +37,7 @@ export function SmartFolderSection() {
const handleEditFolder = async (e: React.MouseEvent, folder: SmartFolder) => {
e.stopPropagation();
setEditFolder(folder);
const automation = await automationStorage.getAutomation(folder.automationId);
const automation = await resolveFolderAutomation(folder);
setEditAutomation(automation);
setCreateModalOpen(true);
};
frontend/src/core/components/smartFolders/SmartFolderWorkbenchView.tsx
+4 -5
View File
@@ -26,8 +26,7 @@ import { useFolderData } from "@app/hooks/useFolderData";
import { useFolderRunState } from "@app/hooks/useFolderRunState";
import { useToolWorkflow } from "@app/contexts/ToolWorkflowContext";
import { SMART_FOLDER_VIEW_ID, SMART_FOLDER_WORKBENCH_ID } from "@app/components/smartFolders/SmartFoldersRegistration";
import { automationStorage } from "@app/services/automationStorage";
import { useFolderAutomation, resolveInputFile } from "@app/hooks/useFolderAutomation";
import { useFolderAutomation, resolveInputFile, resolveFolderAutomation } from "@app/hooks/useFolderAutomation";
import { useLocalFolderPoller } from "@app/hooks/useLocalFolderPoller";
import { AutomationConfig } from "@app/types/automation";
import { iconMap } from "@app/components/tools/automate/iconMap";
@@ -184,10 +183,10 @@ export function SmartFolderWorkbenchView({ data }: SmartFolderWorkbenchViewProps
}, [folderRecord, isServerFolder]);
useEffect(() => {
if (folder?.automationId) {
automationStorage.getAutomation(folder.automationId).then(setAutomation);
if (folder) {
resolveFolderAutomation(folder).then(setAutomation);
}
}, [folder?.automationId]);
}, [folder]);
useEffect(() => {
if (isLocalFolder && folderId) {
frontend/src/core/components/smartFolders/SmartFoldersRegistration.tsx
+4 -2
View File
@@ -1,6 +1,7 @@
import { useEffect, useRef } from 'react';
import { useTranslation } from 'react-i18next';
import { useToolWorkflow } from '@app/contexts/ToolWorkflowContext';
import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext';
import { SmartFolderWorkbenchView } from '@app/components/smartFolders/SmartFolderWorkbenchView';
import { seedDefaultFolders } from '@app/data/smartFolderPresets';
import { useWatchFolderUrlSync } from '@app/hooks/useWatchFolderUrlSync';
@@ -11,6 +12,7 @@ export const SMART_FOLDER_WORKBENCH_ID = 'custom:smartFolder' as const;
export default function SmartFoldersRegistration() {
const { t } = useTranslation();
const { registerCustomWorkbenchView, unregisterCustomWorkbenchView, clearCustomWorkbenchViewData } = useToolWorkflow();
const store = useWatchFolderStore();
useWatchFolderUrlSync();
// Keep refs to latest cleanup callbacks so the registration effect doesn't
@@ -22,8 +24,8 @@ export default function SmartFoldersRegistration() {
useEffect(() => { clearRef.current = clearCustomWorkbenchViewData; });
useEffect(() => {
seedDefaultFolders();
}, []);
seedDefaultFolders(store);
}, [store]);
useEffect(() => {
registerCustomWorkbenchView({
frontend/src/core/contexts/WatchFolderStorageContext.tsx
+12 -18
View File
@@ -7,10 +7,10 @@
*/
import React, { createContext, useContext } from "react";
import { SmartFolder, FolderRecord } from "@app/types/smartFolders";
import { SmartFolderRunEntry } from "@app/types/smartFolders";
import type { SmartFolder, FolderRecord, FolderFileMetadata, SmartFolderRunEntry } from "@app/types/smartFolders";
import { idbBackend } from "@app/services/watchFolderIdbBackend";
// ── Storage interface ──────────────────────────────────────────────────────
// ── Storage interface ─────────────────────────────────────────────────────
export interface WatchFolderStorageBackend {
// Folder CRUD
@@ -23,16 +23,9 @@ export interface WatchFolderStorageBackend {
// File metadata
getFolderData(folderId: string): Promise<FolderRecord | null>;
updateFileMetadata(
folderId: string,
fileId: string,
meta: Partial<import("@app/types/smartFolders").FolderFileMetadata>,
): Promise<void>;
addFileToFolder(
folderId: string,
fileId: string,
meta?: Partial<import("@app/types/smartFolders").FolderFileMetadata>,
): Promise<void>;
updateFileMetadata(folderId: string, fileId: string, meta: Partial<FolderFileMetadata>): Promise<void>;
addFileToFolder(folderId: string, fileId: string, meta?: Partial<FolderFileMetadata>): Promise<void>;
removeFileFromFolder(folderId: string, fileId: string): Promise<void>;
clearFolder(folderId: string): Promise<void>;
// Run state
@@ -44,7 +37,7 @@ export interface WatchFolderStorageBackend {
onChange(callback: () => void): () => void;
}
// ── Context ────────────────────────────────────────────────────────────────
// ── Context ───────────────────────────────────────────────────────────────
const WatchFolderStorageContext = createContext<WatchFolderStorageBackend | null>(null);
@@ -59,9 +52,10 @@ export function WatchFolderStorageProvider({
}
/**
* Returns the storage backend from context, or null if none is provided
* (meaning hooks should fall back to direct IDB imports).
* Returns the storage backend, falling back to `idbBackend` if no provider is mounted.
* The provider IS always mounted in the app, so the fallback is purely defensive
* (e.g. tests rendering hooks in isolation).
*/
export function useWatchFolderStorage(): WatchFolderStorageBackend | null {
return useContext(WatchFolderStorageContext);
export function useWatchFolderStore(): WatchFolderStorageBackend {
return useContext(WatchFolderStorageContext) ?? idbBackend;
}
frontend/src/core/data/smartFolderPresets.ts
+3 -3
View File
@@ -5,7 +5,7 @@
import { SmartFolder } from "@app/types/smartFolders";
import { AutomationConfig } from "@app/types/automation";
import { automationStorage } from "@app/services/automationStorage";
import { smartFolderStorage } from "@app/services/smartFolderStorage";
import type { WatchFolderStorageBackend } from "@app/contexts/WatchFolderStorageContext";
const SEEDED_FLAG = "smart_folders_seeded";
let seedingInProgress = false;
@@ -94,14 +94,14 @@ const PRESETS: PresetDefinition[] = [
},
];
export async function seedDefaultFolders(): Promise<void> {
export async function seedDefaultFolders(store: WatchFolderStorageBackend): Promise<void> {
if (localStorage.getItem(SEEDED_FLAG) || seedingInProgress) return;
seedingInProgress = true;
try {
for (const preset of PRESETS) {
const savedAutomation = await automationStorage.saveAutomation(preset.automation);
await smartFolderStorage.createFolder({
await store.createFolder({
...preset.folder,
automationId: savedAutomation.id,
});
frontend/src/core/hooks/useAllSmartFolders.ts
+5 -11
View File
@@ -6,30 +6,24 @@
import { useState, useEffect } from "react";
import { SmartFolder } from "@app/types/smartFolders";
import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/services/smartFolderStorage";
import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext";
import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext";
export function useAllSmartFolders(): SmartFolder[] {
const backend = useWatchFolderStorage();
const store = useWatchFolderStore();
const [folders, setFolders] = useState<SmartFolder[]>([]);
useEffect(() => {
const load = async () => {
try {
const all = backend ? await backend.getAllFolders() : await smartFolderStorage.getAllFolders();
setFolders(all);
setFolders(await store.getAllFolders());
} catch (err) {
console.error("Failed to load smart folders:", err);
}
};
load();
if (backend) {
return backend.onChange(load);
}
window.addEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, load);
return () => window.removeEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, load);
}, [backend]);
return store.onChange(load);
}, [store]);
return folders;
}
frontend/src/core/hooks/useFolderAutomation.ts
+121 -69
View File
@@ -13,13 +13,12 @@
import { useCallback, useEffect, useRef } from 'react';
import { addSSEHandler, parsePipelineSSEEvent } from '@app/hooks/useSSEConnection';
import { ToolRegistry } from '@app/data/toolsTaxonomy';
import { SmartFolder, isServerFolderInput } from '@app/types/smartFolders';
import { SmartFolder, isServerFolderInput, FolderFileMetadata } from '@app/types/smartFolders';
import { automationStorage } from '@app/services/automationStorage';
import { folderStorage } from '@app/services/folderStorage';
import type { AutomationConfig } from '@app/types/automation';
import { fileStorage } from '@app/services/fileStorage';
import { folderRunStateStorage } from '@app/services/folderRunStateStorage';
import { folderRetryScheduleStorage } from '@app/services/folderRetryScheduleStorage';
import { smartFolderStorage } from '@app/services/smartFolderStorage';
import { useWatchFolderStore, WatchFolderStorageBackend } from '@app/contexts/WatchFolderStorageContext';
import {
executeBackendPipeline,
submitBackendJob,
@@ -80,6 +79,37 @@ function notifySW(message: { type: string }): void {
navigator.serviceWorker?.controller?.postMessage(message);
}
/**
* Resolves the automation to run for a folder.
*
* Server-backed folders carry the pipeline inline as `automationConfig` (the IDB
* automationStorage on this client may not have an entry for `automationId`), so
* prefer that. Otherwise look up by `automationId` in IDB.
*/
export async function resolveFolderAutomation(folder: SmartFolder): Promise<AutomationConfig | null> {
if (folder.automationConfig) {
try {
const parsed = JSON.parse(folder.automationConfig);
const operations = Array.isArray(parsed) ? parsed : parsed.operations;
if (Array.isArray(operations)) {
return {
id: folder.automationId || `inline:${folder.id}`,
name: parsed.name ?? folder.name,
description: parsed.description ?? '',
icon: folder.icon,
operations,
createdAt: folder.createdAt,
updatedAt: folder.updatedAt,
};
}
} catch {
// Fall through to IDB lookup
}
}
if (!folder.automationId) return null;
return automationStorage.getAutomation(folder.automationId);
}
// ---------------------------------------------------------------------------
// Output finalisation — shared by runPipeline (sync fallback) and drainPendingJobs
// ---------------------------------------------------------------------------
@@ -90,13 +120,14 @@ function notifySW(message: { type: string }): void {
* executeBackendPipeline call or an async job poll.
*/
async function finalizeRun(
store: WatchFolderStorageBackend,
folder: SmartFolder,
file: File,
inputFileId: string,
ownedByFolder: boolean,
resultFiles: File[]
): Promise<void> {
const currentFolderData = await folderStorage.getFolderData(folder.id);
const currentFolderData = await store.getFolderData(folder.id);
const currentMeta = currentFolderData?.files[inputFileId];
const prevOutputIds: string[] = currentMeta?.displayFileIds
?? (currentMeta?.displayFileId ? [currentMeta.displayFileId] : []);
@@ -201,14 +232,14 @@ async function finalizeRun(
const processedAt = new Date();
const accumulatedIds = isAutoNumber ? [...prevOutputIds, ...allOutputIds] : allOutputIds;
await folderStorage.updateFileMetadata(folder.id, inputFileId, {
await store.updateFileMetadata(folder.id, inputFileId, {
status: 'processed',
processedAt,
displayFileId: accumulatedIds[0],
displayFileIds: accumulatedIds,
});
await folderRunStateStorage.appendRunEntries(folder.id, [{
await store.addFolderRunEntries(folder.id, [{
inputFileId,
displayFileId: accumulatedIds[0],
displayFileIds: accumulatedIds,
@@ -222,11 +253,12 @@ async function finalizeRun(
// ---------------------------------------------------------------------------
async function findFileByJobId(
store: WatchFolderStorageBackend,
jobId: string
): Promise<{ folder: SmartFolder; fileId: string; meta: import('@app/types/smartFolders').FolderFileMetadata } | null> {
const folders = await smartFolderStorage.getAllFolders();
): Promise<{ folder: SmartFolder; fileId: string; meta: FolderFileMetadata } | null> {
const folders = await store.getAllFolders();
for (const folder of folders) {
const folderData = await folderStorage.getFolderData(folder.id);
const folderData = await store.getFolderData(folder.id);
if (!folderData) continue;
for (const [fileId, meta] of Object.entries(folderData.files)) {
if (meta.serverJobId === jobId) return { folder, fileId, meta };
@@ -252,22 +284,23 @@ async function findFileByJobId(
* executeBackendPipeline (tab-close resilience does not apply to these).
*/
export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
const store = useWatchFolderStore();
const processingRef = useRef<Set<string>>(new Set());
// ── Finalise a job from the SSE handler ───────────────────────────────────
const finalizeFromSSE = useCallback(async (jobId: string, error?: string) => {
const match = await findFileByJobId(jobId);
const match = await findFileByJobId(store, jobId);
if (!match) return;
const { folder, fileId, meta } = match;
if (processingRef.current.has(fileId)) return; // drain already handling it
processingRef.current.add(fileId);
try {
const freshMeta = (await folderStorage.getFolderData(folder.id))?.files[fileId];
const freshMeta = (await store.getFolderData(folder.id))?.files[fileId];
if (freshMeta?.status !== 'processing') return; // already finalised by drain
if (error) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: error,
serverJobId: undefined,
@@ -277,7 +310,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
const inputFile = await fileStorage.getStirlingFile(fileId as FileId);
if (!inputFile) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: 'Input file missing from storage',
serverJobId: undefined,
@@ -285,10 +318,10 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
return;
}
const resultFiles = await getBackendJobResult(jobId, folder.name);
await finalizeRun(folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles);
await folderStorage.updateFileMetadata(folder.id, fileId, { serverJobId: undefined });
await finalizeRun(store, folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles);
await store.updateFileMetadata(folder.id, fileId, { serverJobId: undefined });
} catch (err) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: err instanceof Error ? err.message : 'Failed to retrieve job result',
serverJobId: undefined,
@@ -296,7 +329,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
} finally {
processingRef.current.delete(fileId);
}
}, []);
}, [store]);
// ── Server-folder SSE completion handler ──────────────────────────────────
// Output filenames are "{fileId}.{ext}" — strip extension to get the IDB fileId directly.
@@ -306,9 +339,9 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
folderId: string,
outputFilenames: string[]
) => {
const folder = await smartFolderStorage.getFolder(folderId);
const folder = await store.getFolder(folderId);
if (!folder || folder.isPaused) return;
const folderData = await folderStorage.getFolderData(folderId);
const folderData = await store.getFolderData(folderId);
if (!folderData) return;
for (const outputFilename of outputFilenames) {
@@ -321,19 +354,19 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
if (processingRef.current.has(fileId)) continue;
processingRef.current.add(fileId);
try {
const freshMeta = (await folderStorage.getFolderData(folderId))?.files[fileId];
const freshMeta = (await store.getFolderData(folderId))?.files[fileId];
if (freshMeta?.status !== 'processing') continue; // already finalised
const processedAt = new Date();
// Record completion — output lives on the server, not in IDB file storage
await folderStorage.updateFileMetadata(folderId, fileId, {
await store.updateFileMetadata(folderId, fileId, {
status: 'processed',
processedAt,
serverOutputFilenames: [outputFilename],
pendingOnServerFolder: undefined,
});
await folderRunStateStorage.appendRunEntries(folderId, [{
await store.addFolderRunEntries(folderId, [{
inputFileId: fileId,
displayFileId: fileId,
processedAt,
@@ -363,7 +396,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
}
}
} catch (err) {
await folderStorage.updateFileMetadata(folderId, fileId, {
await store.updateFileMetadata(folderId, fileId, {
status: 'error',
errorMessage: err instanceof Error ? err.message : 'Failed to finalize server output',
pendingOnServerFolder: undefined,
@@ -372,7 +405,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
processingRef.current.delete(fileId);
}
}
}, []);
}, [store]);
// ── Server-folder SSE error handler ───────────────────────────────────────
// Marks files as failed when PipelineDirectoryProcessor reports a batch error.
@@ -384,7 +417,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
if (processingRef.current.has(fileId)) continue;
processingRef.current.add(fileId);
try {
await folderStorage.updateFileMetadata(folderId, fileId, {
await store.updateFileMetadata(folderId, fileId, {
status: 'error',
errorMessage: 'Server-side processing failed',
pendingOnServerFolder: undefined,
@@ -393,17 +426,17 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
processingRef.current.delete(fileId);
}
}
}, []);
}, [store]);
// ── Recovery drain — runs once on mount/visibility/SW wake ────────────────
// Only needed when SSE was down during job completion (tab close, server restart).
// On transient network errors: leave as 'processing' — drain retries next cycle.
// On 404: job expired from server — mark as error.
const drainPendingJobs = useCallback(async () => {
const folders = await smartFolderStorage.getAllFolders();
const folders = await store.getAllFolders();
for (const folder of folders) {
if (folder.isPaused) continue;
const folderData = await folderStorage.getFolderData(folder.id);
const folderData = await store.getFolderData(folder.id);
if (!folderData) continue;
for (const [fileId, meta] of Object.entries(folderData.files)) {
@@ -420,7 +453,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
// 404 → job expired (server restarted / TTL hit) — surface as error
// Anything else → transient network issue, leave as 'processing' for next drain
if ((err as any)?.response?.status === 404) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: 'Job expired — server may have restarted. Retry to reprocess.',
serverJobId: undefined,
@@ -432,7 +465,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
if (jobStatus.status === 'completed') {
const inputFile = await fileStorage.getStirlingFile(fileId as FileId);
if (!inputFile) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: 'Input file missing from storage',
serverJobId: undefined,
@@ -441,17 +474,17 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
}
try {
const resultFiles = await getBackendJobResult(jobId, folder.name);
await finalizeRun(folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles);
await folderStorage.updateFileMetadata(folder.id, fileId, { serverJobId: undefined });
await finalizeRun(store, folder, inputFile, fileId, meta.ownedByFolder ?? false, resultFiles);
await store.updateFileMetadata(folder.id, fileId, { serverJobId: undefined });
} catch (err) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: err instanceof Error ? err.message : 'Failed to retrieve job result',
serverJobId: undefined,
});
}
} else if (jobStatus.status === 'failed') {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: jobStatus.error || 'Server job failed',
serverJobId: undefined,
@@ -485,12 +518,12 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
if (processingRef.current.has(fileId)) continue;
processingRef.current.add(fileId);
try {
const freshMeta = (await folderStorage.getFolderData(folder.id))?.files[fileId];
const freshMeta = (await store.getFolderData(folder.id))?.files[fileId];
if (freshMeta?.status !== 'processing') continue;
const inputFile = await fileStorage.getStirlingFile(fileId as FileId);
if (!inputFile) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: 'Input file missing from storage',
pendingOnServerFolder: undefined,
@@ -498,13 +531,13 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
continue;
}
const processedAt = new Date();
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'processed',
processedAt,
serverOutputFilenames: [outputFile.filename],
pendingOnServerFolder: undefined,
});
await folderRunStateStorage.appendRunEntries(folder.id, [{
await store.addFolderRunEntries(folder.id, [{
inputFileId: fileId,
displayFileId: fileId,
processedAt,
@@ -529,7 +562,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
} catch { /* best-effort */ }
}
} catch (err) {
await folderStorage.updateFileMetadata(folder.id, fileId, {
await store.updateFileMetadata(folder.id, fileId, {
status: 'error',
errorMessage: err instanceof Error ? err.message : 'Failed to retrieve server output',
pendingOnServerFolder: undefined,
@@ -539,7 +572,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
}
}
}
}, []);
}, [store]);
// ── Core pipeline runner ───────────────────────────────────────────────────
const runPipeline = useCallback(
@@ -553,16 +586,16 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
processingRef.current.add(inputFileId);
try {
const automation = await automationStorage.getAutomation(folder.automationId);
const automation = await resolveFolderAutomation(folder);
if (!automation) {
await folderStorage.updateFileMetadata(folder.id, inputFileId, {
await store.updateFileMetadata(folder.id, inputFileId, {
status: 'error',
errorMessage: 'Automation not found',
});
return;
}
await folderStorage.updateFileMetadata(folder.id, inputFileId, { status: 'processing' });
await store.updateFileMetadata(folder.id, inputFileId, { status: 'processing' });
// Server-folder input — upload to watch folder, trigger immediate processing via SSE
if (isServerFolderInput(folder)) {
@@ -571,7 +604,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
// is no longer readable after the async resolveInputFile call completed.
const uploadFile = await fileStorage.getStirlingFile(inputFileId as FileId) ?? file;
await uploadFileToServerFolder(folder.id, inputFileId, uploadFile);
await folderStorage.updateFileMetadata(folder.id, inputFileId, {
await store.updateFileMetadata(folder.id, inputFileId, {
pendingOnServerFolder: true,
});
// Fire-and-forget trigger — don't wait for processing to start
@@ -584,7 +617,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
const jobId = await submitBackendJob(automation, [file], toolRegistry);
if (jobId !== null) {
await folderStorage.updateFileMetadata(folder.id, inputFileId, { serverJobId: jobId });
await store.updateFileMetadata(folder.id, inputFileId, { serverJobId: jobId });
// Release lock — SSE handler / drain will re-acquire when finalising
processingRef.current.delete(inputFileId);
return;
@@ -592,10 +625,10 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
// Sync fallback (automation contains a custom-processor step)
const resultFiles = await executeBackendPipeline(automation, [file], toolRegistry);
await finalizeRun(folder, file, inputFileId, ownedByFolder, resultFiles);
await finalizeRun(store, folder, file, inputFileId, ownedByFolder, resultFiles);
} catch (err: unknown) {
const existing = await folderStorage.getFolderData(folder.id);
const existing = await store.getFolderData(folder.id);
const prev = existing?.files[inputFileId];
const attempts = (prev?.failedAttempts ?? 0) + 1;
const maxRetries = folder.maxRetries ?? 3;
@@ -603,7 +636,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
const willRetry = maxRetries > 0 && attempts < maxRetries && retryDelayMs > 0;
const nextRetryAt = willRetry ? Date.now() + retryDelayMs : undefined;
await folderStorage.updateFileMetadata(folder.id, inputFileId, {
await store.updateFileMetadata(folder.id, inputFileId, {
status: 'error',
errorMessage: err instanceof Error ? err.message : 'Unknown error',
failedAttempts: attempts,
@@ -624,10 +657,10 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
} finally {
// Safety net: if still 'processing' without a serverJobId, something went wrong
try {
const record = await folderStorage.getFolderData(folder.id);
const record = await store.getFolderData(folder.id);
const fileMeta = record?.files[inputFileId];
if (fileMeta?.status === 'processing' && !fileMeta?.serverJobId && !fileMeta?.pendingOnServerFolder) {
await folderStorage.updateFileMetadata(folder.id, inputFileId, {
await store.updateFileMetadata(folder.id, inputFileId, {
status: 'error',
errorMessage: 'Processing failed unexpectedly',
});
@@ -638,14 +671,14 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
processingRef.current.delete(inputFileId);
}
},
[toolRegistry]
[toolRegistry, store]
);
// ── Sync server-folder sessions on mount ──────────────────────────────────
// Ensures the server's session.json always points to the current browser session,
// so SSE notifications are routed here even after localStorage was cleared.
const syncServerFolderSessions = useCallback(async () => {
const folders = await smartFolderStorage.getAllFolders();
const folders = await store.getAllFolders();
for (const folder of folders) {
if (!isServerFolderInput(folder)) continue;
try {
@@ -655,9 +688,9 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
// when the folder was created). Re-provision it now using the stored automation.
if (err?.response?.status === 404) {
try {
const automation = await automationStorage.getAutomation(folder.automationId);
const automation = await resolveFolderAutomation(folder);
if (!automation) {
console.warn(`[watch-folders] Cannot re-provision ${folder.id}: automation ${folder.automationId} not found in IDB`);
console.warn(`[watch-folders] Cannot re-provision ${folder.id}: no automation (id=${folder.automationId}, hasInlineConfig=${!!folder.automationConfig})`);
} else {
const configJson = buildPipelineJson(automation, toolRegistry);
if (!configJson) {
@@ -678,29 +711,47 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
}
}
}
}, [toolRegistry]);
}, [toolRegistry, store]);
// ── Lifecycle effects ──────────────────────────────────────────────────────
// Mirror callbacks into refs so the lifecycle effect can mount listeners once
// (SSE / SW / visibility) without tearing them down whenever callback identities
// change. The cost of re-registering the SSE handler is dropping in-flight events.
const storeRef = useRef(store);
const runPipelineRef = useRef(runPipeline);
const drainPendingJobsRef = useRef(drainPendingJobs);
const finalizeFromSSERef = useRef(finalizeFromSSE);
const finalizeFromServerFolderSSERef = useRef(finalizeFromServerFolderSSE);
const finalizeServerFolderErrorRef = useRef(finalizeServerFolderError);
const syncServerFolderSessionsRef = useRef(syncServerFolderSessions);
useEffect(() => { storeRef.current = store; }, [store]);
useEffect(() => { runPipelineRef.current = runPipeline; }, [runPipeline]);
useEffect(() => { drainPendingJobsRef.current = drainPendingJobs; }, [drainPendingJobs]);
useEffect(() => { finalizeFromSSERef.current = finalizeFromSSE; }, [finalizeFromSSE]);
useEffect(() => { finalizeFromServerFolderSSERef.current = finalizeFromServerFolderSSE; }, [finalizeFromServerFolderSSE]);
useEffect(() => { finalizeServerFolderErrorRef.current = finalizeServerFolderError; }, [finalizeServerFolderError]);
useEffect(() => { syncServerFolderSessionsRef.current = syncServerFolderSessions; }, [syncServerFolderSessions]);
useEffect(() => {
async function drainDueRetries() {
const due = await folderRetryScheduleStorage.claimDue();
for (const entry of due) {
const freshFolder = await smartFolderStorage.getFolder(entry.folderId);
const freshFolder = await storeRef.current.getFolder(entry.folderId);
if (!freshFolder || freshFolder.isPaused) continue;
const freshFile = await fileStorage.getStirlingFile(entry.fileId as FileId);
if (!freshFile) continue;
await folderStorage.updateFileMetadata(entry.folderId, entry.fileId, {
await storeRef.current.updateFileMetadata(entry.folderId, entry.fileId, {
status: 'pending',
nextRetryAt: undefined,
serverJobId: undefined,
});
void runPipeline(freshFolder, freshFile, entry.fileId, entry.ownedByFolder);
void runPipelineRef.current(freshFolder, freshFile, entry.fileId, entry.ownedByFolder);
}
}
void drainDueRetries();
void drainPendingJobs();
void syncServerFolderSessions();
void drainPendingJobsRef.current();
void syncServerFolderSessionsRef.current();
if ('serviceWorker' in navigator) {
navigator.serviceWorker
@@ -710,26 +761,27 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
function handleSWMessage(event: MessageEvent) {
if (event.data?.type === 'PROCESS_DUE_RETRIES') void drainDueRetries();
if (event.data?.type === 'POLL_PIPELINE_JOBS') void drainPendingJobs();
if (event.data?.type === 'POLL_PIPELINE_JOBS') void drainPendingJobsRef.current();
}
navigator.serviceWorker?.addEventListener('message', handleSWMessage);
function handleVisibilityChange() {
if (document.visibilityState === 'visible') {
void drainDueRetries();
void drainPendingJobs();
void drainPendingJobsRef.current();
}
}
document.addEventListener('visibilitychange', handleVisibilityChange);
// SSE handler — receives job-complete / job-failed / server-folder-complete / server-folder-error events
// SSE handler — receives job-complete / job-failed / server-folder-complete / server-folder-error events.
// Reads callbacks from refs so identity changes don't force re-subscription.
const removeSSEHandler = addSSEHandler((data: unknown) => {
const event = parsePipelineSSEEvent(data);
if (!event) return;
if (event.type === 'job-complete') void finalizeFromSSE(event.jobId);
if (event.type === 'job-failed') void finalizeFromSSE(event.jobId, event.error ?? 'Server job failed');
if (event.type === 'server-folder-complete') void finalizeFromServerFolderSSE(event.folderId, event.outputFiles);
if (event.type === 'server-folder-error') void finalizeServerFolderError(event.folderId, event.failedFileIds);
if (event.type === 'job-complete') void finalizeFromSSERef.current(event.jobId);
if (event.type === 'job-failed') void finalizeFromSSERef.current(event.jobId, event.error ?? 'Server job failed');
if (event.type === 'server-folder-complete') void finalizeFromServerFolderSSERef.current(event.folderId, event.outputFiles);
if (event.type === 'server-folder-error') void finalizeServerFolderErrorRef.current(event.folderId, event.failedFileIds);
});
return () => {
@@ -737,7 +789,7 @@ export function useFolderAutomation(toolRegistry: Partial<ToolRegistry>) {
document.removeEventListener('visibilitychange', handleVisibilityChange);
removeSSEHandler();
};
}, [runPipeline, drainPendingJobs, finalizeFromSSE, finalizeFromServerFolderSSE, finalizeServerFolderError, syncServerFolderSessions]);
}, []);
/** Run multiple files through the pipeline concurrently. */
const processBatch = useCallback(
frontend/src/core/hooks/useFolderData.ts
+14 -24
View File
@@ -5,7 +5,7 @@
import { useState, useEffect, useCallback, useMemo } from "react";
import { FolderFileMetadata, FolderRecord } from "@app/types/smartFolders";
import { folderStorage } from "@app/services/folderStorage";
import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext";
import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext";
interface UseFolderDataReturn {
folderRecord: FolderRecord | null;
@@ -23,23 +23,25 @@ interface UseFolderDataReturn {
}
export function useFolderData(folderId: string): UseFolderDataReturn {
const backend = useWatchFolderStorage();
const store = useWatchFolderStore();
const [folderRecord, setFolderRecord] = useState<FolderRecord | null>(null);
const refresh = useCallback(async () => {
if (!folderId) return;
try {
const record = backend ? await backend.getFolderData(folderId) : await folderStorage.getFolderData(folderId);
const record = await store.getFolderData(folderId);
setFolderRecord(record);
} catch (error) {
console.error("Failed to load folder data:", error);
}
}, [folderId, backend]);
}, [folderId, store]);
useEffect(() => {
refresh();
}, [refresh]);
// Subscribe to IDB change events — the server backend also writes to IDB,
// so this picks up changes from both local and server-backed operations.
useEffect(() => {
const unsubscribe = folderStorage.onFolderChange((changedFolderId) => {
if (changedFolderId === folderId) {
@@ -57,40 +59,28 @@ export function useFolderData(folderId: string): UseFolderDataReturn {
const addFile = useCallback(
async (fileId: string, metadata?: Partial<FolderFileMetadata>) => {
if (backend) {
await backend.addFileToFolder(folderId, fileId, metadata);
} else {
await folderStorage.addFileToFolder(folderId, fileId, metadata);
}
await store.addFileToFolder(folderId, fileId, metadata);
},
[folderId, backend],
[folderId, store],
);
const removeFile = useCallback(
async (fileId: string) => {
await folderStorage.removeFileFromFolder(folderId, fileId);
await store.removeFileFromFolder(folderId, fileId);
},
[folderId],
[folderId, store],
);
const updateFileMetadata = useCallback(
async (fileId: string, updates: Partial<FolderFileMetadata>) => {
if (backend) {
await backend.updateFileMetadata(folderId, fileId, updates);
} else {
await folderStorage.updateFileMetadata(folderId, fileId, updates);
}
await store.updateFileMetadata(folderId, fileId, updates);
},
[folderId, backend],
[folderId, store],
);
const clearFolder = useCallback(async () => {
if (backend) {
await backend.clearFolder(folderId);
} else {
await folderStorage.clearFolder(folderId);
}
}, [folderId, backend]);
await store.clearFolder(folderId);
}, [folderId, store]);
const getFileMetadata = useCallback(
(fileId: string): FolderFileMetadata | null => {
frontend/src/core/hooks/useFolderMembership.ts
+5 -2
View File
@@ -7,9 +7,11 @@
import { useState, useEffect } from 'react';
import { folderStorage } from '@app/services/folderStorage';
import { useAllSmartFolders } from '@app/hooks/useAllSmartFolders';
import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext';
export function useFolderMembership(): Map<string, string[]> {
const folders = useAllSmartFolders();
const store = useWatchFolderStore();
const [membership, setMembership] = useState<Map<string, string[]>>(new Map());
useEffect(() => {
@@ -27,7 +29,7 @@ export function useFolderMembership(): Map<string, string[]> {
};
for (const folder of folders) {
try {
const record = await folderStorage.getFolderData(folder.id);
const record = await store.getFolderData(folder.id);
if (record) {
Object.entries(record.files).forEach(([fileId, meta]) => {
add(fileId, folder.id);
@@ -45,8 +47,9 @@ export function useFolderMembership(): Map<string, string[]> {
load();
// Server backend mirrors writes to IDB so this fires for both backends.
return folderStorage.onFolderChange(load);
}, [folders]);
}, [folders, store]);
return membership;
}
frontend/src/core/hooks/useFolderOutputIds.ts
+5 -2
View File
@@ -6,9 +6,11 @@
import { useState, useEffect } from 'react';
import { folderStorage } from '@app/services/folderStorage';
import { useAllSmartFolders } from '@app/hooks/useAllSmartFolders';
import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext';
export function useFolderOutputIds(): Set<string> {
const folders = useAllSmartFolders();
const store = useWatchFolderStore();
const [outputIds, setOutputIds] = useState<Set<string>>(new Set());
useEffect(() => {
@@ -18,7 +20,7 @@ export function useFolderOutputIds(): Set<string> {
const ids = new Set<string>();
for (const folder of folders) {
try {
const record = await folderStorage.getFolderData(folder.id);
const record = await store.getFolderData(folder.id);
if (record) {
Object.values(record.files).forEach(meta => {
const oids = meta?.displayFileIds ?? (meta?.displayFileId ? [meta.displayFileId] : []);
@@ -31,8 +33,9 @@ export function useFolderOutputIds(): Set<string> {
};
load();
// Server backend mirrors writes to IDB so this fires for both backends.
return folderStorage.onFolderChange(load);
}, [folders]);
}, [folders, store]);
return outputIds;
}
frontend/src/core/hooks/useFolderRunState.ts
+9 -15
View File
@@ -5,50 +5,44 @@
import { useState, useEffect, useCallback } from 'react';
import { SmartFolderRunEntry } from '@app/types/smartFolders';
import { folderRunStateStorage } from '@app/services/folderRunStateStorage';
import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext';
interface UseFolderRunStateReturn {
recentRuns: SmartFolderRunEntry[];
setRecentRuns: (runs: SmartFolderRunEntry[]) => Promise<void>;
clearRecentRuns: () => Promise<void>;
isLoading: boolean;
}
export function useFolderRunState(folderId: string): UseFolderRunStateReturn {
const store = useWatchFolderStore();
const [recentRuns, setRecentRunsState] = useState<SmartFolderRunEntry[]>([]);
const [isLoading, setIsLoading] = useState(true);
// Subscribe before loading to close the race window between load completing
// and the listener being established (writes in that gap would be missed).
// Server backend mirrors writes to IDB so this fires for both backends.
useEffect(() => {
if (!folderId) return;
setIsLoading(true);
const unsub = folderRunStateStorage.onRunStateChange((changedFolderId) => {
if (changedFolderId !== folderId) return;
folderRunStateStorage
store
.getFolderRunState(folderId)
.then(setRecentRunsState)
.catch((err) => console.error('Failed to reload folder run state:', err));
});
folderRunStateStorage
store
.getFolderRunState(folderId)
.then(setRecentRunsState)
.catch((err) => console.error('Failed to load folder run state:', err))
.finally(() => setIsLoading(false));
return unsub;
}, [folderId]);
const setRecentRuns = useCallback(
async (runs: SmartFolderRunEntry[]) => {
await folderRunStateStorage.setFolderRunState(folderId, runs);
setRecentRunsState(runs);
},
[folderId]
);
}, [folderId, store]);
const clearRecentRuns = useCallback(async () => {
await folderRunStateStorage.clearFolderRunState(folderId);
await store.clearFolderRunState(folderId);
setRecentRunsState([]);
}, [folderId]);
}, [folderId, store]);
return { recentRuns, setRecentRuns, clearRecentRuns, isLoading };
return { recentRuns, clearRecentRuns, isLoading };
}
frontend/src/core/hooks/useFolderRunStatuses.ts
+10 -8
View File
@@ -6,7 +6,11 @@
import { useState, useEffect, useRef } from "react";
import { SmartFolder, SmartFolderRunEntry } from "@app/types/smartFolders";
import { folderRunStateStorage } from "@app/services/folderRunStateStorage";
import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext";
import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext";
// IDB run-state events fire for both backends — the server backend mirrors writes to IDB
// (see serverBackend.addFolderRunEntries), so listening to the IDB event surface gives us
// live updates for both local and server-backed folders.
export type FolderRunStatus = "idle" | "processing" | "done";
@@ -23,7 +27,7 @@ function deriveStatus(runs: SmartFolderRunEntry[]): FolderRunStatus {
}
export function useFolderRunStatuses(folders: SmartFolder[]): Record<string, FolderRunStatus> {
const backend = useWatchFolderStorage();
const store = useWatchFolderStore();
const [statuses, setStatuses] = useState<Record<string, FolderRunStatus>>({});
const doneTimersRef = useRef<Map<string, ReturnType<typeof setTimeout>>>(new Map());
const foldersRef = useRef(folders);
@@ -36,9 +40,7 @@ export function useFolderRunStatuses(folders: SmartFolder[]): Record<string, Fol
const results = await Promise.all(
folders.map(async (folder) => {
try {
const runs = backend
? await backend.getFolderRunState(folder.id)
: await folderRunStateStorage.getFolderRunState(folder.id);
const runs = await store.getFolderRunState(folder.id);
return [folder.id, deriveStatus(runs)] as const;
} catch {
return [folder.id, "idle" as FolderRunStatus] as const;
@@ -53,20 +55,20 @@ export function useFolderRunStatuses(folders: SmartFolder[]): Record<string, Fol
};
load();
}, [folders]);
}, [folders, store]);
// Update individual folder status live when new run entries are appended
useEffect(() => {
return folderRunStateStorage.onRunStateChange((changedFolderId) => {
if (!foldersRef.current.find((f) => f.id === changedFolderId)) return;
folderRunStateStorage
store
.getFolderRunState(changedFolderId)
.then((runs) => {
setStatuses((prev) => ({ ...prev, [changedFolderId]: deriveStatus(runs) }));
})
.catch((err) => console.error("Failed to update run status:", err));
});
}, []);
}, [store]);
// When a folder becomes 'done', revert to 'idle' after TTL
useEffect(() => {
frontend/src/core/hooks/useLocalFolderPoller.ts
+7 -5
View File
@@ -8,8 +8,7 @@
import { useEffect, useRef } from 'react';
import { SmartFolder } from '@app/types/smartFolders';
import { smartFolderStorage } from '@app/services/smartFolderStorage';
import { folderStorage } from '@app/services/folderStorage';
import { useWatchFolderStore } from '@app/contexts/WatchFolderStorageContext';
import { folderDirectoryHandleStorage } from '@app/services/folderDirectoryHandleStorage';
import { folderSeenFilesStorage, makeSeenKey } from '@app/services/folderSeenFilesStorage';
import { resolveInputFile } from '@app/hooks/useFolderAutomation';
@@ -20,8 +19,11 @@ const POLL_INTERVAL_MS = 10_000;
export function useLocalFolderPoller(
runPipeline: (folder: SmartFolder, file: File, inputFileId: string, ownedByFolder: boolean) => Promise<void>
): void {
const store = useWatchFolderStore();
const runPipelineRef = useRef(runPipeline);
const storeRef = useRef(store);
useEffect(() => { runPipelineRef.current = runPipeline; });
useEffect(() => { storeRef.current = store; });
useEffect(() => {
let cancelled = false;
@@ -31,7 +33,7 @@ export function useLocalFolderPoller(
let folders: SmartFolder[];
try {
folders = await smartFolderStorage.getAllFolders();
folders = await storeRef.current.getAllFolders();
} catch {
return;
}
@@ -48,7 +50,7 @@ export function useLocalFolderPoller(
const hasPermission = await folderDirectoryHandleStorage.ensureReadPermission(inputHandle);
if (!hasPermission) continue;
const folderData = await folderStorage.getFolderData(folder.id);
const folderData = await storeRef.current.getFolderData(folder.id);
// Build set of file names already in the folder (any status) to avoid duplicates
// keyed by name+size (can't use lastModified — file handle gives same value each time)
const processingNames = new Set(
@@ -84,7 +86,7 @@ export function useLocalFolderPoller(
const { inputFileId, ownedByFolder } = await resolveInputFile(file);
await folderStorage.addFileToFolder(folder.id, inputFileId, {
await storeRef.current.addFileToFolder(folder.id, inputFileId, {
status: 'pending',
name: file.name,
ownedByFolder,
frontend/src/core/hooks/useSmartFolders.ts
+18 -48
View File
@@ -4,15 +4,13 @@
import { useState, useEffect, useCallback } from "react";
import { SmartFolder, isServerFolderInput } from "@app/types/smartFolders";
import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/services/smartFolderStorage";
import { folderStorage } from "@app/services/folderStorage";
import { folderRunStateStorage } from "@app/services/folderRunStateStorage";
import { fileStorage } from "@app/services/fileStorage";
import { deleteServerFolder } from "@app/services/serverFolderApiService";
import { folderRetryScheduleStorage } from "@app/services/folderRetryScheduleStorage";
import { folderSeenFilesStorage } from "@app/services/folderSeenFilesStorage";
import { folderDirectoryHandleStorage } from "@app/services/folderDirectoryHandleStorage";
import { useWatchFolderStorage } from "@app/contexts/WatchFolderStorageContext";
import { useWatchFolderStore } from "@app/contexts/WatchFolderStorageContext";
import { FileId } from "@app/types/fileContext";
interface UseSmartFoldersReturn {
@@ -25,13 +23,12 @@ interface UseSmartFoldersReturn {
}
export function useSmartFolders(): UseSmartFoldersReturn {
const backend = useWatchFolderStorage();
const store = useWatchFolderStore();
const [folders, setFolders] = useState<SmartFolder[]>([]);
const [loading, setLoading] = useState(true);
const refreshFolders = useCallback(async () => {
try {
const store = backend ?? { getAllFolders: () => smartFolderStorage.getAllFolders() };
const all = await store.getAllFolders();
setFolders(all);
} catch (error) {
@@ -39,62 +36,45 @@ export function useSmartFolders(): UseSmartFoldersReturn {
} finally {
setLoading(false);
}
}, [backend]);
}, [store]);
useEffect(() => {
refreshFolders();
}, [refreshFolders]);
useEffect(() => {
if (backend) {
return backend.onChange(() => refreshFolders());
}
const handler = () => {
refreshFolders();
};
window.addEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, handler);
return () => window.removeEventListener(SMART_FOLDER_STORAGE_CHANGE_EVENT, handler);
}, [backend, refreshFolders]);
return store.onChange(() => refreshFolders());
}, [store, refreshFolders]);
const createFolder = useCallback(
async (data: Omit<SmartFolder, "id" | "createdAt" | "updatedAt">): Promise<SmartFolder> => {
if (backend) return backend.createFolder(data);
return smartFolderStorage.createFolder(data);
return store.createFolder(data);
},
[backend],
[store],
);
const updateFolder = useCallback(
async (folder: SmartFolder): Promise<SmartFolder> => {
if (backend) return backend.updateFolder(folder);
return smartFolderStorage.updateFolder(folder);
return store.updateFolder(folder);
},
[backend],
[store],
);
const deleteFolder = useCallback(
async (id: string): Promise<void> => {
const store = backend ?? {
getFolder: (fid: string) => smartFolderStorage.getFolder(fid),
deleteFolder: (fid: string) => smartFolderStorage.deleteFolder(fid),
getFolderData: (fid: string) => folderStorage.getFolderData(fid),
clearFolder: (fid: string) => folderStorage.clearFolder(fid),
clearFolderRunState: (fid: string) => folderRunStateStorage.clearFolderRunState(fid),
};
// Clean up server watch folder first (best-effort — don't block if server is down)
// Clean up server watch folder first (best-effort)
const folderMeta = await store.getFolder(id);
if (folderMeta && isServerFolderInput(folderMeta)) {
await deleteServerFolder(id).catch(() => {});
}
const record = await (backend ? backend.getFolderData(id) : folderStorage.getFolderData(id));
const record = await store.getFolderData(id);
if (record) {
// Only delete input files the folder created from disk — never touch sidebar-sourced files.
// Only delete input files the folder created from disk
const ownedInputIds = Object.entries(record.files)
.filter(([, meta]) => meta.ownedByFolder === true)
.map(([fid]) => fid);
// Always delete every output the folder produced (folder always owns those).
// Always delete every output the folder produced
const outputIds = Object.values(record.files).flatMap(
(meta) => meta.displayFileIds ?? (meta.displayFileId ? [meta.displayFileId] : []),
);
@@ -102,28 +82,18 @@ export function useSmartFolders(): UseSmartFoldersReturn {
await Promise.all(toDelete.map((fid) => fileStorage.deleteStirlingFile(fid as FileId).catch(() => {})));
}
if (backend) {
await backend.clearFolder(id);
await backend.clearFolderRunState(id);
} else {
await folderStorage.clearFolder(id);
await folderRunStateStorage.clearFolderRunState(id);
}
await store.clearFolder(id);
await store.clearFolderRunState(id);
// These are always IDB-only (browser API objects / ephemeral state)
await folderRetryScheduleStorage.clearFolder(id).catch(() => {});
await folderSeenFilesStorage.clearFolder(id).catch(() => {});
await folderDirectoryHandleStorage.remove(id).catch(() => {});
await folderDirectoryHandleStorage.removeInput(id).catch(() => {});
if (backend) {
await backend.deleteFolder(id);
} else {
await smartFolderStorage.deleteFolder(id);
}
await store.deleteFolder(id);
// Notify the sidebar file list that files have been removed.
window.dispatchEvent(new CustomEvent("stirling:files-changed"));
},
[backend],
[store],
);
return { folders, loading, createFolder, updateFolder, deleteFolder, refreshFolders };
frontend/src/core/services/folderStorage.ts
+8 -3
View File
@@ -153,15 +153,20 @@ class FolderStorage {
});
}
/** Overwrite the entire folder record (used by sync from server). */
async setFolderData(folderId: string, record: FolderRecord): Promise<void> {
/**
* Overwrite the entire folder record (used by sync from server).
* Pass `{ silent: true }` for server-mirror writes — these reflect a read,
* not a user action, so dispatching change events would cause subscribers
* (which themselves call getFolderData) to re-fetch in an infinite loop.
*/
async setFolderData(folderId: string, record: FolderRecord, opts?: { silent?: boolean }): Promise<void> {
const db = await this.ensureDB();
return new Promise((resolve, reject) => {
const transaction = db.transaction([this.recordsStore], "readwrite");
const store = transaction.objectStore(this.recordsStore);
const request = store.put(record);
request.onsuccess = () => {
this.dispatchChange(folderId);
if (!opts?.silent) this.dispatchChange(folderId);
resolve();
};
request.onerror = () => reject(new Error("Failed to set folder data"));
frontend/src/core/services/watchFolderIdbBackend.ts
+1 -1
View File
@@ -9,7 +9,6 @@ import type { WatchFolderStorageBackend } from "@app/contexts/WatchFolderStorage
import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/services/smartFolderStorage";
import { folderStorage } from "@app/services/folderStorage";
import { folderRunStateStorage } from "@app/services/folderRunStateStorage";
import type { SmartFolder, FolderRecord, FolderFileMetadata, SmartFolderRunEntry } from "@app/types/smartFolders";
export const idbBackend: WatchFolderStorageBackend = {
// Folder CRUD
@@ -24,6 +23,7 @@ export const idbBackend: WatchFolderStorageBackend = {
getFolderData: (folderId) => folderStorage.getFolderData(folderId),
updateFileMetadata: (folderId, fileId, meta) => folderStorage.updateFileMetadata(folderId, fileId, meta),
addFileToFolder: (folderId, fileId, meta) => folderStorage.addFileToFolder(folderId, fileId, meta),
removeFileFromFolder: (folderId, fileId) => folderStorage.removeFileFromFolder(folderId, fileId),
clearFolder: (folderId) => folderStorage.clearFolder(folderId),
// Run state
frontend/src/core/types/smartFolders.ts
+17 -10
View File
@@ -14,16 +14,16 @@ export interface SmartFolder {
order?: number;
isDefault?: boolean;
isPaused?: boolean;
maxRetries?: number; // 0 = disabled; default 3
maxRetries?: number; // 0 = disabled; default 3
retryDelayMinutes?: number; // default 5
outputMode?: 'new_file' | 'new_version'; // default: 'new_file' (existing behaviour)
outputName?: string; // output filename prefix/suffix
outputNamePosition?: 'prefix' | 'suffix' | 'auto-number'; // default: 'prefix'
hasOutputDirectory?: boolean; // true when a local FS output folder is configured
outputMode?: "new_file" | "new_version"; // default: 'new_file' (existing behaviour)
outputName?: string; // output filename prefix/suffix
outputNamePosition?: "prefix" | "suffix" | "auto-number"; // default: 'prefix'
hasOutputDirectory?: boolean; // true when a local FS output folder is configured
/** Where input files come from. Default: 'idb' (dropped/sidebar files stay in browser). */
inputSource?: 'idb' | 'local-folder' | 'server-folder';
inputSource?: "idb" | "local-folder" | "server-folder";
/** Where processing happens. Default: 'local' (browser). Forced to 'server' when inputSource='server-folder'. */
processingMode?: 'local' | 'server';
processingMode?: "local" | "server";
/**
* How long to keep output files in the server's processed/ dir (hours).
* null / undefined = keep forever. Only meaningful when inputSource='server-folder'.
@@ -34,11 +34,18 @@ export interface SmartFolder {
* immediately after downloading it. Only meaningful when inputSource='server-folder'.
*/
deleteOutputOnDownload?: boolean;
/** Visibility: PERSONAL (owner-only) or ORGANISATION (all users). Only set when server-backed. */
scope?: "PERSONAL" | "ORGANISATION";
/**
* Inlined automation pipeline (JSON-stringified operations array).
* Used by the server DB instead of automationId. When present, takes precedence.
*/
automationConfig?: string;
}
export interface FolderFileMetadata {
addedAt: Date;
status: 'pending' | 'processing' | 'processed' | 'error';
status: "pending" | "processing" | "processed" | "error";
processedAt?: Date;
/** All output file ids produced by this run — references stirling-pdf-files */
displayFileIds?: string[];
@@ -64,7 +71,7 @@ export interface FolderFileMetadata {
/** Type guard / helper — true when the folder's input source is the server watch folder. */
export function isServerFolderInput(folder: SmartFolder): boolean {
return folder.inputSource === 'server-folder';
return folder.inputSource === "server-folder";
}
export interface FolderRecord {
@@ -81,5 +88,5 @@ export interface SmartFolderRunEntry {
displayFileIds?: string[];
/** When this run completed — used for TTL-based "done" status */
processedAt?: Date;
status: 'processing' | 'processed';
status: "processing" | "processed";
}
frontend/src/proprietary/components/WatchFolderServerProvider.tsx
+95 -3
View File
@@ -1,22 +1,114 @@
/**
* Proprietary wrapper that overrides the core IDB-only WatchFolderStorageProvider
* with the server-backed implementation when premium is enabled.
*
* On first premium load, migrates existing IDB folders, file metadata, and run
* history to the server. The flag is only set when ALL upserts succeed; partial
* failures cause a retry on the next load.
*/
import React from "react";
import React, { useEffect, useRef } from "react";
import { useAppConfig } from "@app/contexts/AppConfigContext";
import { WatchFolderStorageProvider } from "@app/contexts/WatchFolderStorageContext";
import { serverBackend } from "@proprietary/services/watchFolderServerBackend";
import { smartFolderStorage } from "@app/services/smartFolderStorage";
import { folderStorage } from "@app/services/folderStorage";
import { folderRunStateStorage } from "@app/services/folderRunStateStorage";
import { watchFolderApi } from "@proprietary/services/watchFolderApiService";
const MIGRATION_KEY = "watch_folders_migrated_to_server";
const MIGRATION_LOCK = "watch-folders-migration";
async function migrateOne(folderId: string, serverIds: Set<string>): Promise<boolean> {
const folder = await smartFolderStorage.getFolder(folderId);
if (!folder) return true;
// 1. Folder definition (skip if already on server — another tab/device may have done it)
if (!serverIds.has(folder.id)) {
await serverBackend.createFolderWithId(folder);
}
// 2. File metadata
const record = await folderStorage.getFolderData(folder.id);
if (record) {
for (const [fileId, meta] of Object.entries(record.files)) {
await serverBackend.addFileToFolder(folder.id, fileId, meta);
// updateFileMetadata pushes the full snapshot; addFileToFolder only pushes the basics.
await serverBackend.updateFileMetadata(folder.id, fileId, meta);
}
}
// 3. Run history
const runs = await folderRunStateStorage.getFolderRunState(folder.id);
if (runs.length > 0) {
await serverBackend.addFolderRunEntries(folder.id, runs);
}
return true;
}
async function runMigration(): Promise<boolean> {
if (localStorage.getItem(MIGRATION_KEY)) return true;
const idbFolders = await smartFolderStorage.getAllFolders();
if (idbFolders.length === 0) {
localStorage.setItem(MIGRATION_KEY, "1");
return true;
}
const serverFolders = await watchFolderApi.list();
const serverIds = new Set(serverFolders.map((f) => f.id));
let allOk = true;
for (const folder of idbFolders) {
try {
await migrateOne(folder.id, serverIds);
} catch (err) {
console.warn(`[watch-folders] Migration of folder ${folder.id} failed:`, err);
allOk = false;
}
}
// Only commit the flag when every folder migrated cleanly; otherwise retry next load.
if (allOk) localStorage.setItem(MIGRATION_KEY, "1");
return allOk;
}
export function WatchFolderServerProvider({ children }: { children: React.ReactNode }) {
const { config } = useAppConfig();
const isPremium = config?.premiumEnabled === true;
const migrationRan = useRef(false);
useEffect(() => {
if (!isPremium || migrationRan.current) return;
if (localStorage.getItem(MIGRATION_KEY)) return;
migrationRan.current = true;
// Use Web Locks API when available so concurrent tabs don't double-migrate;
// fall back to a plain run when the API is missing (older browsers / SSR).
const run = async () => {
try {
await runMigration();
} catch (err) {
console.warn("[watch-folders] Migration aborted:", err);
}
};
if ("locks" in navigator) {
navigator.locks.request(MIGRATION_LOCK, { ifAvailable: true }, async (lock) => {
// ifAvailable: lock is null if another tab has it — skip; that tab will commit the flag.
if (!lock) return;
if (localStorage.getItem(MIGRATION_KEY)) return; // re-check after lock acquired
await run();
});
} else {
void run();
}
}, [isPremium]);
if (!isPremium) {
// Core's IDB provider is already in place — just pass through
return <>{children}</>;
}
// Override with server-backed storage
return <WatchFolderStorageProvider backend={serverBackend}>{children}</WatchFolderStorageProvider>;
}
frontend/src/proprietary/services/watchFolderApiService.ts
+8
View File
@@ -102,6 +102,10 @@ export const watchFolderApi = {
await apiClient.delete(`${BASE}/${folderId}/files`);
},
async deleteFile(folderId: string, fileId: string): Promise<void> {
await apiClient.delete(`${BASE}/${folderId}/files/${encodeURIComponent(fileId)}`);
},
// Runs
async listRuns(folderId: string): Promise<WatchFolderRunDTO[]> {
const res = await apiClient.get<WatchFolderRunDTO[]>(`${BASE}/${folderId}/runs`);
@@ -117,4 +121,8 @@ export const watchFolderApi = {
const res = await apiClient.post<WatchFolderRunDTO[]>(`${BASE}/${folderId}/runs/batch`, runs);
return res.data;
},
async deleteRuns(folderId: string): Promise<void> {
await apiClient.delete(`${BASE}/${folderId}/runs`);
},
};
frontend/src/proprietary/services/watchFolderServerBackend.ts
+111 -36
View File
@@ -3,7 +3,7 @@
*
* Writes go to the server API first (source of truth), then mirror to IDB for fast reads.
* Reads come from IDB (populated on init and after writes).
* Falls back to IDB-only if the server is unreachable.
* Falls back to IDB-only on network errors (NOT on auth/permission errors).
*/
import type { WatchFolderStorageBackend } from "@app/contexts/WatchFolderStorageContext";
@@ -11,16 +11,38 @@ import { smartFolderStorage, SMART_FOLDER_STORAGE_CHANGE_EVENT } from "@app/serv
import { folderStorage } from "@app/services/folderStorage";
import { folderRunStateStorage } from "@app/services/folderRunStateStorage";
import type { SmartFolder, FolderRecord, FolderFileMetadata, SmartFolderRunEntry } from "@app/types/smartFolders";
import { watchFolderApi, WatchFolderDTO, WatchFolderFileDTO, WatchFolderRunDTO } from "./watchFolderApiService";
import { watchFolderApi, WatchFolderDTO } from "./watchFolderApiService";
import { AxiosError } from "axios";
// ── DTO ↔ Domain conversions ───────────────────────────────────────────────
// ── Error classification ──────────────────────────────────────────────────
/** Returns true only for network/timeout errors where fallback to IDB makes sense. */
function isNetworkError(err: unknown): boolean {
if (err instanceof AxiosError) {
// No response at all = network failure / timeout
if (!err.response) return true;
// 5xx = server error, safe to fall back
if (err.response.status >= 500) return true;
// 401/403 = auth issue — don't silently fall back to stale IDB data
return false;
}
return true; // Unknown error type — treat as network
}
/** True if the error is a 404 — treat as "not found" rather than a fatal error. */
function isNotFound(err: unknown): boolean {
return err instanceof AxiosError && err.response?.status === 404;
}
// ── DTO ↔ Domain conversions ──────────────────────────────────────────────
function toSmartFolder(dto: WatchFolderDTO): SmartFolder {
return {
id: dto.id,
name: dto.name,
description: dto.description ?? "",
automationId: "", // automation config is inlined — hooks resolve this
automationId: "", // server uses inlined automationConfig instead
automationConfig: dto.automationConfig,
icon: dto.icon ?? "FolderIcon",
accentColor: dto.accentColor ?? "#3b82f6",
createdAt: dto.createdAt ?? new Date().toISOString(),
@@ -28,6 +50,7 @@ function toSmartFolder(dto: WatchFolderDTO): SmartFolder {
order: dto.orderIndex,
isDefault: dto.isDefault,
isPaused: dto.isPaused,
scope: dto.scope,
inputSource: (dto.inputSource as SmartFolder["inputSource"]) ?? "idb",
processingMode: (dto.processingMode as SmartFolder["processingMode"]) ?? "local",
outputMode: (dto.outputMode as SmartFolder["outputMode"]) ?? "new_file",
@@ -40,14 +63,15 @@ function toSmartFolder(dto: WatchFolderDTO): SmartFolder {
};
}
function toDTO(folder: SmartFolder & { scope?: string }): WatchFolderDTO {
function toDTO(folder: SmartFolder): WatchFolderDTO {
return {
id: folder.id,
name: folder.name,
description: folder.description,
automationConfig: folder.automationConfig,
icon: folder.icon,
accentColor: folder.accentColor,
scope: (folder as any).scope ?? "PERSONAL",
scope: folder.scope ?? "PERSONAL",
orderIndex: folder.order,
isDefault: folder.isDefault,
isPaused: folder.isPaused,
@@ -67,11 +91,22 @@ function dispatchChange() {
window.dispatchEvent(new Event(SMART_FOLDER_STORAGE_CHANGE_EVENT));
}
// ── Sync helper: pull server state into IDB ────────────────────────────────
// ── Sync helper: pull server state into IDB ───────────────────────────────
let lastSyncAt = 0;
const SYNC_DEBOUNCE_MS = 10_000; // Don't re-sync more than once per 10s
async function syncFoldersToIdb(force = false): Promise<SmartFolder[]> {
const now = Date.now();
if (!force && now - lastSyncAt < SYNC_DEBOUNCE_MS) {
// Return IDB cache — it's fresh enough
return smartFolderStorage.getAllFolders();
}
async function syncFoldersToIdb(): Promise<SmartFolder[]> {
const dtos = await watchFolderApi.list();
const folders = dtos.map(toSmartFolder);
lastSyncAt = Date.now();
// Overwrite IDB with server state
const existing = await smartFolderStorage.getAllFolders();
const serverIds = new Set(folders.map((f) => f.id));
@@ -89,14 +124,14 @@ async function syncFoldersToIdb(): Promise<SmartFolder[]> {
return folders;
}
// ── Backend implementation ─────────────────────────────────────────────────
// ── Backend implementation ────────────────────────────────────────────────
export const serverBackend: WatchFolderStorageBackend = {
async getAllFolders() {
try {
return await syncFoldersToIdb();
} catch {
// Server unreachable — fall back to IDB
} catch (err) {
if (!isNetworkError(err)) throw err;
return smartFolderStorage.getAllFolders();
}
},
@@ -107,7 +142,11 @@ export const serverBackend: WatchFolderStorageBackend = {
const folder = toSmartFolder(dto);
await smartFolderStorage.createFolderWithId(folder).catch(() => {});
return folder;
} catch {
} catch (err) {
// 404 → folder genuinely doesn't exist on server (or was deleted in another tab).
// Don't fall back to IDB; return null so callers can treat as gone.
if (isNotFound(err)) return null;
if (!isNetworkError(err)) throw err;
return smartFolderStorage.getFolder(id);
}
},
@@ -124,10 +163,11 @@ export const serverBackend: WatchFolderStorageBackend = {
const created = await watchFolderApi.create(toDTO(folder));
const result = toSmartFolder(created);
await smartFolderStorage.createFolderWithId(result).catch(() => {});
lastSyncAt = 0; // Invalidate sync cache
dispatchChange();
return result;
} catch {
// Offline — create in IDB only
} catch (err) {
if (!isNetworkError(err)) throw err;
return smartFolderStorage.createFolder(data);
}
},
@@ -137,9 +177,11 @@ export const serverBackend: WatchFolderStorageBackend = {
const created = await watchFolderApi.create(toDTO(folder));
const result = toSmartFolder(created);
await smartFolderStorage.createFolderWithId(result).catch(() => {});
lastSyncAt = 0;
dispatchChange();
return result;
} catch {
} catch (err) {
if (!isNetworkError(err)) throw err;
return smartFolderStorage.createFolderWithId(folder);
}
},
@@ -149,9 +191,11 @@ export const serverBackend: WatchFolderStorageBackend = {
const updated = await watchFolderApi.update(folder.id, toDTO(folder));
const result = toSmartFolder(updated);
await smartFolderStorage.createFolderWithId(result).catch(() => {});
lastSyncAt = 0;
dispatchChange();
return result;
} catch {
} catch (err) {
if (!isNetworkError(err)) throw err;
return smartFolderStorage.updateFolder(folder);
}
},
@@ -159,8 +203,14 @@ export const serverBackend: WatchFolderStorageBackend = {
async deleteFolder(id) {
try {
await watchFolderApi.remove(id);
} catch {
// best-effort server delete
lastSyncAt = 0;
} catch (err) {
// 404 = already deleted — that's success for delete semantics.
if (isNotFound(err)) {
lastSyncAt = 0;
} else if (!isNetworkError(err)) {
throw err;
}
}
await smartFolderStorage.deleteFolder(id);
},
@@ -188,10 +238,13 @@ export const serverBackend: WatchFolderStorageBackend = {
processedAt: f.processedAt ? new Date(f.processedAt) : undefined,
};
}
// Mirror to IDB
await folderStorage.setFolderData(folderId, record).catch(() => {});
// Mirror to IDB silently — this is a cache write reflecting a read, not a user action.
// Dispatching FOLDER_CHANGE_EVENT here would cause subscribers (useFolderData, etc.)
// to re-call getFolderData → server → mirror → … infinite loop.
await folderStorage.setFolderData(folderId, record, { silent: true }).catch(() => {});
return record;
} catch {
} catch (err) {
if (!isNetworkError(err)) throw err;
return folderStorage.getFolderData(folderId);
}
},
@@ -199,7 +252,7 @@ export const serverBackend: WatchFolderStorageBackend = {
async updateFileMetadata(folderId, fileId, meta) {
// Update IDB immediately for fast UI
await folderStorage.updateFileMetadata(folderId, fileId, meta);
// Sync to server
// Sync to server — only swallow network failures; surface auth/config errors.
try {
const existing = await folderStorage.getFolderData(folderId);
const fileMeta = existing?.files[fileId];
@@ -218,8 +271,8 @@ export const serverBackend: WatchFolderStorageBackend = {
processedAt: fileMeta.processedAt?.toISOString(),
});
}
} catch {
// Server sync failed — IDB is still up to date
} catch (err) {
if (!isNetworkError(err)) throw err;
}
},
@@ -233,16 +286,33 @@ export const serverBackend: WatchFolderStorageBackend = {
ownedByFolder: meta?.ownedByFolder,
addedAt: meta?.addedAt?.toISOString() ?? new Date().toISOString(),
});
} catch {
// offline — IDB has the data
} catch (err) {
if (!isNetworkError(err)) throw err;
}
},
async removeFileFromFolder(folderId, fileId) {
await folderStorage.removeFileFromFolder(folderId, fileId);
try {
await watchFolderApi.deleteFile(folderId, fileId);
} catch (err) {
if (isNotFound(err)) {
// Already gone server-side — DELETE is idempotent.
} else if (!isNetworkError(err)) {
throw err;
}
}
},
async clearFolder(folderId) {
try {
await watchFolderApi.deleteFiles(folderId);
} catch {
/* best-effort */
} catch (err) {
if (isNotFound(err)) {
// Already cleared on server — proceed.
} else if (!isNetworkError(err)) {
throw err;
}
}
await folderStorage.clearFolder(folderId);
},
@@ -259,7 +329,8 @@ export const serverBackend: WatchFolderStorageBackend = {
processedAt: r.processedAt ? new Date(r.processedAt) : undefined,
}));
return entries;
} catch {
} catch (err) {
if (!isNetworkError(err)) throw err;
return folderRunStateStorage.getFolderRunState(folderId);
}
},
@@ -279,18 +350,22 @@ export const serverBackend: WatchFolderStorageBackend = {
processedAt: e.processedAt?.toISOString(),
})),
);
} catch {
/* offline */
} catch (err) {
if (!isNetworkError(err)) throw err;
}
},
async clearFolderRunState(folderId) {
try {
// No dedicated endpoint yet — runs are deleted with the folder
} catch {
/* best-effort */
}
await folderRunStateStorage.clearFolderRunState(folderId);
try {
await watchFolderApi.deleteRuns(folderId);
} catch (err) {
if (isNotFound(err)) {
// Folder already gone or no runs — fine.
} else if (!isNetworkError(err)) {
throw err;
}
}
},
onChange(callback) {