feat(documents): prepare inbox suggestions and add entity assignments

backend/src/modules/scanner-inbox.service.ts

@@ -1,12 +1,13 @@
 import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"
-import { and, asc, count, desc, eq, ilike, isNull, sql } from "drizzle-orm"
+import { and, asc, count, desc, eq, ilike, inArray, isNull, sql } from "drizzle-orm"
 import type { FastifyInstance } from "fastify"
 import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
 import { files, filetags, folders, scannerInboxConnections, scannerInboxDocuments, tenants } from "../../db/schema"
 import { sanitizeFilename } from "../utils/filename"
 import { s3 } from "../utils/s3"
 import { secrets } from "../utils/secrets"
-import { suggestDocumentFiling, validateFilingSuggestion, type FilingSuggestionResult } from "./scanner-inbox-suggestions"
+import { suggestDocumentFiling, validateFilingSuggestion } from "./scanner-inbox-suggestions"
+import { assignmentKinds, assignmentOptions, currentAssignments, validateAssignments, type FileAssignments } from "./scanner-inbox-assignments"
 
 export const SCANNER_UPLOAD_LIMIT = 100 * 1024 * 1024
 export const SCANNER_DAV_PATH = "/scanner-inbox/"
@@ -47,7 +48,6 @@ export function validateScannerFile(filename: string, body: Buffer) {
 }
 
 export function createScannerInboxService(db: FastifyInstance["db"], storage: ScannerStorage = objectStorage, analyze = suggestDocumentFiling) {
-    const pendingSuggestions = new Map<string, Promise<FilingSuggestionResult>>()
     const publicConnection = {
         username: scannerInboxConnections.username,
         active: scannerInboxConnections.active,
@@ -93,13 +93,19 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
             isNull(scannerInboxDocuments.filedAt), eq(files.archived, false),
             search ? ilike(scannerInboxDocuments.remoteFilename, `%${search.replace(/[\\%_]/g, "\\$&")}%`) : undefined)
         const documents = await db.select({ id: files.id, name: scannerInboxDocuments.remoteFilename,
-            mimeType: files.mimeType, size: files.size, receivedAt: scannerInboxDocuments.receivedAt })
+            mimeType: files.mimeType, size: files.size, receivedAt: scannerInboxDocuments.receivedAt, suggestionStatus: scannerInboxDocuments.suggestionStatus })
             .from(scannerInboxDocuments).innerJoin(files, eq(files.id, scannerInboxDocuments.fileId))
             .where(condition).orderBy(desc(scannerInboxDocuments.receivedAt), desc(files.id)).limit(limit).offset(offset)
         const [total] = await db.select({ value: count() }).from(scannerInboxDocuments)
             .innerJoin(files, eq(files.id, scannerInboxDocuments.fileId)).where(condition)
         return { documents, total: total.value }
     }
+    const openCount = async (tenantId: number) => {
+        const [row] = await db.select({ total: count() }).from(scannerInboxDocuments)
+            .innerJoin(files, eq(files.id, scannerInboxDocuments.fileId))
+            .where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(files.tenant, tenantId), isNull(scannerInboxDocuments.filedAt), eq(files.archived, false)))
+        return row
+    }
     const filingOptions = async (tenantId: number) => {
         const [availableFolders, types] = await Promise.all([
             db.select({ id: folders.id, name: folders.name, parent: folders.parent,
@@ -108,33 +114,77 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
             db.select({ id: filetags.id, name: filetags.name }).from(filetags)
                 .where(and(eq(filetags.tenant, tenantId), eq(filetags.archived, false))).orderBy(asc(filetags.name)),
         ])
-        return { folders: availableFolders, types }
+        return { folders: availableFolders, types, assignmentKinds }
     }
     const suggestionDocument = async (tenantId: number, fileId: string) => {
-        const [document] = await db.select({ path: files.path, name: scannerInboxDocuments.remoteFilename, mimeType: files.mimeType, size: files.size })
+        const [document] = await db.select({ path: files.path, name: scannerInboxDocuments.remoteFilename, mimeType: files.mimeType, size: files.size,
+            status: scannerInboxDocuments.suggestionStatus, suggestion: scannerInboxDocuments.suggestion, message: scannerInboxDocuments.suggestionMessage })
             .from(scannerInboxDocuments).innerJoin(files, eq(files.id, scannerInboxDocuments.fileId))
             .where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(files.tenant, tenantId), eq(files.id, fileId),
                 isNull(scannerInboxDocuments.filedAt), eq(files.archived, false))).limit(1)
         if (!document?.path) throw scannerError(404, "Dokument nicht in der Ablage gefunden.")
         return document
     }
-    const suggestFiling = async (tenantId: number, fileId: string): Promise<FilingSuggestionResult> => {
-        // Check tenant ownership before reading storage or submitting anything to the AI provider.
+    const getSuggestion = async (tenantId: number, fileId: string) => {
         const document = await suggestionDocument(tenantId, fileId)
-        const key = `${tenantId}:${fileId}`
-        if (pendingSuggestions.has(key)) return pendingSuggestions.get(key)!
-        if (pendingSuggestions.size >= 4) throw scannerError(429, "Die KI verarbeitet gerade weitere Dokumente. Bitte gleich erneut versuchen.")
-        const operation = (async () => {
-            const result = await analyze(document, await filingOptions(tenantId))
-            await suggestionDocument(tenantId, fileId)
-            const suggestion = validateFilingSuggestion(result, await filingOptions(tenantId))
-            return { suggestion, message: suggestion ? undefined : result.message || "Keine passende Zuordnung verfügbar. Bitte manuell auswählen." }
-        })()
-        pendingSuggestions.set(key, operation)
-        try { return await operation }
-        finally { pendingSuggestions.delete(key) }
+        const suggestion = document.status === "ready"
+            ? validateFilingSuggestion({ suggestion: document.suggestion }, await filingOptions(tenantId)) : null
+        return { status: document.status, suggestion, message: document.status === "ready" && !suggestion
+            ? document.message || "Keine passende Zuordnung verfügbar. Bitte manuell auswählen." : document.message }
     }
-    const fileDocument = async (tenantId: number, userId: string, fileId: string, folderId: string | null, typeId: string) => db.transaction(async tx => {
+    const queueSuggestion = async (tenantId: number, fileId: string) => {
+        await suggestionDocument(tenantId, fileId)
+        await db.update(scannerInboxDocuments).set({ suggestionStatus: "pending", suggestion: null, suggestionMessage: null,
+            suggestionClaim: null, suggestionUpdatedAt: new Date() })
+            .where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(scannerInboxDocuments.fileId, fileId),
+                isNull(scannerInboxDocuments.filedAt), inArray(scannerInboxDocuments.suggestionStatus, ["ready", "failed"])))
+        return getSuggestion(tenantId, fileId)
+    }
+    const processNextSuggestion = async () => {
+        const claim = randomUUID()
+        // Claim without holding a transaction open while OCR/the provider runs. Expired claims survive restarts.
+        const claimed = await db.execute<{ fileId: string; tenantId: string }>(sql`
+            UPDATE scanner_inbox_documents AS job
+            SET suggestion_status = 'processing', suggestion_claim = ${claim}, suggestion_updated_at = now()
+            WHERE job.file_id = (
+                SELECT pending.file_id FROM scanner_inbox_documents AS pending
+                JOIN files ON files.id = pending.file_id AND files.tenant = pending.tenant_id
+                JOIN tenants ON tenants.id = pending.tenant_id
+                WHERE pending.filed_at IS NULL AND files.archived = false
+                    AND tenants.locked IS DISTINCT FROM 'maintenance_tenant'
+                    AND (pending.suggestion_status = 'pending' OR
+                        (pending.suggestion_status = 'processing' AND pending.suggestion_updated_at < now() - interval '5 minutes'))
+                ORDER BY pending.received_at, pending.file_id FOR UPDATE OF pending SKIP LOCKED LIMIT 1
+            ) RETURNING job.file_id AS "fileId", job.tenant_id AS "tenantId"`)
+        const job = claimed.rows[0]
+        if (!job) return false
+        const tenantId = Number(job.tenantId)
+        const condition = and(eq(scannerInboxDocuments.fileId, job.fileId), eq(scannerInboxDocuments.tenantId, tenantId),
+            eq(scannerInboxDocuments.suggestionClaim, claim), isNull(scannerInboxDocuments.filedAt))
+        try {
+            const document = await suggestionDocument(tenantId, job.fileId)
+            const { path, name, mimeType, size } = document
+            const result = await analyze({ path, name, mimeType, size }, await filingOptions(tenantId))
+            await suggestionDocument(tenantId, job.fileId)
+            const suggestion = validateFilingSuggestion(result, await filingOptions(tenantId))
+            await db.update(scannerInboxDocuments).set({ suggestionStatus: "ready", suggestion,
+                suggestionMessage: suggestion ? null : result.message || "Keine passende Zuordnung gefunden. Bitte manuell auswählen.",
+                suggestionClaim: null, suggestionUpdatedAt: new Date() }).where(condition)
+        } catch (error: any) {
+            const message = error?.statusCode === 503 ? "Die KI-Erkennung ist noch nicht konfiguriert. Bitte manuell zuordnen oder später erneut prüfen."
+                : "Die KI-Vorbereitung ist fehlgeschlagen. Bitte erneut prüfen oder manuell zuordnen."
+            await db.update(scannerInboxDocuments).set({ suggestionStatus: "failed", suggestion: null, suggestionMessage: message,
+                suggestionClaim: null, suggestionUpdatedAt: new Date() }).where(condition)
+        }
+        return true
+    }
+    const getAssignments = async (tenantId: number, fileId: string) => {
+        await suggestionDocument(tenantId, fileId)
+        const [file] = await db.select().from(files).where(and(eq(files.id, fileId), eq(files.tenant, tenantId), eq(files.archived, false))).limit(1)
+        if (!file) throw scannerError(404, "Dokument nicht gefunden.")
+        return currentAssignments(db, tenantId, file)
+    }
+    const fileDocument = async (tenantId: number, userId: string, fileId: string, folderId: string | null, typeId: string, assignments: FileAssignments = {}) => db.transaction(async tx => {
         const [tenant] = await tx.select({ locked: tenants.locked }).from(tenants)
             .where(eq(tenants.id, tenantId)).for("share")
         if (!tenant) throw scannerError(404, "Mandant nicht gefunden.")
@@ -157,11 +207,12 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
         const [type] = await tx.select({ id: filetags.id }).from(filetags)
             .where(and(eq(filetags.id, typeId), eq(filetags.tenant, tenantId), eq(filetags.archived, false))).for("share")
         if (!type) throw scannerError(400, "Der gewählte Dokumenttyp ist nicht verfügbar.")
+        const assignmentChanges = await validateAssignments(tx, tenantId, assignments)
         const now = new Date()
-        await tx.update(files).set({ folder: folderId, type: typeId, updatedAt: now, updatedBy: userId })
+        await tx.update(files).set({ ...assignmentChanges, folder: folderId, type: typeId, updatedAt: now, updatedBy: userId })
             .where(and(eq(files.id, fileId), eq(files.tenant, tenantId)))
         // Keep the receipt so a scanner retry cannot recreate a document after filing.
-        await tx.update(scannerInboxDocuments).set({ filedAt: now })
+        await tx.update(scannerInboxDocuments).set({ filedAt: now, suggestionClaim: null })
             .where(and(eq(scannerInboxDocuments.fileId, fileId), eq(scannerInboxDocuments.tenantId, tenantId)))
         return { id: fileId, folder: folderId, type: typeId }
     })
@@ -225,7 +276,8 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
             throw error
         }
     }
-    return { connection, credentials, disable, authenticate, find, list, filingOptions, suggestFiling, fileDocument, probe, accept }
+    return { connection, credentials, disable, authenticate, find, list, openCount, filingOptions, getSuggestion, queueSuggestion, processNextSuggestion,
+        getAssignments, assignmentOptions: (tenantId: number, kind: string, search: string) => assignmentOptions(db, tenantId, kind, search), fileDocument, probe, accept }
 }
 
 export type ScannerInboxService = ReturnType<typeof createScannerInboxService>