feat(documents): prepare inbox suggestions and add entity assignments
This commit is contained in:
13
backend/db/migrations/0077_scanner_inbox_preparation.sql
Normal file
13
backend/db/migrations/0077_scanner_inbox_preparation.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
ALTER TABLE "scanner_inbox_documents"
|
||||
ADD COLUMN "suggestion_status" text NOT NULL DEFAULT 'pending',
|
||||
ADD COLUMN "suggestion" jsonb,
|
||||
ADD COLUMN "suggestion_message" text,
|
||||
ADD COLUMN "suggestion_claim" uuid,
|
||||
ADD COLUMN "suggestion_updated_at" timestamptz NOT NULL DEFAULT now();
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "scanner_inbox_documents" ADD CONSTRAINT "scanner_inbox_suggestion_status_check"
|
||||
CHECK ("suggestion_status" IN ('pending', 'processing', 'ready', 'failed'));
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "scanner_inbox_suggestion_queue_idx"
|
||||
ON "scanner_inbox_documents" ("suggestion_status", "suggestion_updated_at", "received_at")
|
||||
WHERE "filed_at" IS NULL;
|
||||
@@ -519,6 +519,13 @@
|
||||
"when": 1790726400001,
|
||||
"tag": "0076_collaboration_federation",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 74,
|
||||
"version": "7",
|
||||
"when": 1790899200000,
|
||||
"tag": "0077_scanner_inbox_preparation",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { bigint, boolean, index, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"
|
||||
import { bigint, boolean, index, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"
|
||||
import { tenants } from "./tenants"
|
||||
import { files } from "./files"
|
||||
import { sql } from "drizzle-orm"
|
||||
|
||||
export const scannerInboxConnections = pgTable("scanner_inbox_connections", {
|
||||
tenantId: bigint("tenant_id", { mode: "number" }).primaryKey().references(() => tenants.id, { onDelete: "cascade" }),
|
||||
@@ -18,7 +19,13 @@ export const scannerInboxDocuments = pgTable("scanner_inbox_documents", {
|
||||
checksum: text("checksum").notNull(),
|
||||
receivedAt: timestamp("received_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
filedAt: timestamp("filed_at", { withTimezone: true }),
|
||||
suggestionStatus: text("suggestion_status").notNull().default("pending"),
|
||||
suggestion: jsonb("suggestion"),
|
||||
suggestionMessage: text("suggestion_message"),
|
||||
suggestionClaim: uuid("suggestion_claim"),
|
||||
suggestionUpdatedAt: timestamp("suggestion_updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
}, table => ({
|
||||
filenameKey: uniqueIndex("scanner_inbox_documents_filename_key").on(table.tenantId, table.remoteFilename),
|
||||
receivedIdx: index("scanner_inbox_documents_received_idx").on(table.tenantId, table.receivedAt),
|
||||
suggestionQueueIdx: index("scanner_inbox_suggestion_queue_idx").on(table.suggestionStatus, table.suggestionUpdatedAt, table.receivedAt).where(sql`${table.filedAt} IS NULL`),
|
||||
}))
|
||||
|
||||
@@ -69,6 +69,7 @@ import {initS3} from "./utils/s3";
|
||||
import { runBootstrap } from "./modules/bootstrap.service";
|
||||
import { startCentralServicesHeartbeat } from "./modules/central-services-heartbeat.service";
|
||||
import { startDocumentImportWorker } from "./modules/document-import/document-import.worker";
|
||||
import { startScannerInboxWorker } from "./modules/scanner-inbox.worker";
|
||||
|
||||
|
||||
//Services
|
||||
@@ -97,6 +98,7 @@ async function main() {
|
||||
await runBootstrap(app);
|
||||
startCentralServicesHeartbeat(app);
|
||||
startDocumentImportWorker(app);
|
||||
startScannerInboxWorker(app);
|
||||
|
||||
app.addHook('preHandler', (req, reply, done) => {
|
||||
console.log(req.method)
|
||||
|
||||
70
backend/src/modules/scanner-inbox-assignments.ts
Normal file
70
backend/src/modules/scanner-inbox-assignments.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { and, asc, eq, ilike, isNull, sql, type SQL } from "drizzle-orm"
|
||||
import type { AnyPgColumn, AnyPgTable } from "drizzle-orm/pg-core"
|
||||
import type { FastifyInstance } from "fastify"
|
||||
import { authProfiles, checks, contracts, createddocuments, customers, incominginvoices, inventoryitems, plants, products, projects, vehicles, vendors } from "../../db/schema"
|
||||
|
||||
type Target = { label: string; table: AnyPgTable; id: AnyPgColumn; tenant: AnyPgColumn; name: AnyPgColumn; available: SQL; uuid?: boolean }
|
||||
export const assignmentTargets: Record<string, Target> = {
|
||||
project: { label: "Projekt", table: projects, id: projects.id, tenant: projects.tenant, name: projects.name, available: eq(projects.archived, false) },
|
||||
customer: { label: "Kunde", table: customers, id: customers.id, tenant: customers.tenant, name: customers.name, available: eq(customers.archived, false) },
|
||||
vendor: { label: "Lieferant", table: vendors, id: vendors.id, tenant: vendors.tenant, name: vendors.name, available: eq(vendors.archived, false) },
|
||||
vehicle: { label: "Fahrzeug", table: vehicles, id: vehicles.id, tenant: vehicles.tenant, name: vehicles.license_plate, available: eq(vehicles.archived, false) },
|
||||
plant: { label: "Objekt", table: plants, id: plants.id, tenant: plants.tenant, name: plants.name, available: eq(plants.archived, false) },
|
||||
contract: { label: "Vertrag", table: contracts, id: contracts.id, tenant: contracts.tenant, name: contracts.name, available: eq(contracts.archived, false) },
|
||||
product: { label: "Produkt", table: products, id: products.id, tenant: products.tenant, name: products.name, available: eq(products.archived, false) },
|
||||
createddocument: { label: "Ausgangsbeleg", table: createddocuments, id: createddocuments.id, tenant: createddocuments.tenant, name: createddocuments.documentNumber, available: eq(createddocuments.archived, false) },
|
||||
incominginvoice: { label: "Eingangsrechnung", table: incominginvoices, id: incominginvoices.id, tenant: incominginvoices.tenant, name: incominginvoices.reference, available: eq(incominginvoices.archived, false) },
|
||||
inventoryitem: { label: "Inventarartikel", table: inventoryitems, id: inventoryitems.id, tenant: inventoryitems.tenant, name: inventoryitems.name, available: eq(inventoryitems.archived, false) },
|
||||
check: { label: "Überprüfung", table: checks, id: checks.id, tenant: checks.tenant, name: checks.name, available: eq(checks.archived, false), uuid: true },
|
||||
authProfile: { label: "Mitarbeiter", table: authProfiles, id: authProfiles.id, tenant: authProfiles.tenant_id, name: authProfiles.full_name,
|
||||
available: and(eq(authProfiles.active, true), isNull(authProfiles.customer_for_portal)), uuid: true },
|
||||
}
|
||||
export const assignmentKinds = Object.entries(assignmentTargets).map(([value, target]) => ({ value, label: target.label }))
|
||||
export type FileAssignments = Record<string, string | number | null>
|
||||
const invalidAssignment = () => Object.assign(new Error("Die gewählte Zuordnung ist nicht verfügbar."), { statusCode: 400 })
|
||||
const targetFor = (kind: string) => {
|
||||
if (!Object.prototype.hasOwnProperty.call(assignmentTargets, kind)) throw invalidAssignment()
|
||||
return assignmentTargets[kind]
|
||||
}
|
||||
export function assignmentId(kind: string, id: unknown) {
|
||||
const target = targetFor(kind)
|
||||
if (target.uuid) {
|
||||
if (typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) throw invalidAssignment()
|
||||
return id
|
||||
}
|
||||
if (!/^[1-9]\d*$/.test(String(id)) || !Number.isSafeInteger(Number(id))) throw invalidAssignment()
|
||||
return Number(id)
|
||||
}
|
||||
type Reader = Pick<FastifyInstance["db"], "select">
|
||||
export async function assignmentOptions(db: Reader, tenantId: number, kind: string, search = "") {
|
||||
const target = targetFor(kind)
|
||||
const name = sql<string>`coalesce(nullif(${target.name}, ''), '#' || ${target.id}::text)`
|
||||
const rows = await db.select({ id: target.id, name }).from(target.table)
|
||||
.where(and(eq(target.tenant, tenantId), target.available,
|
||||
search ? ilike(name, `%${search.replace(/[\\%_]/g, "\\$&")}%`) : undefined))
|
||||
.orderBy(asc(target.name), asc(target.id)).limit(100)
|
||||
return rows.map(row => ({ id: String(row.id), name: row.name }))
|
||||
}
|
||||
export async function currentAssignments(db: Reader, tenantId: number, file: Record<string, unknown>) {
|
||||
const assignments = []
|
||||
for (const [kind, target] of Object.entries(assignmentTargets)) {
|
||||
if (file[kind] == null) continue
|
||||
const [entry] = await db.select({ name: target.name }).from(target.table)
|
||||
.where(and(eq(target.id, file[kind]), eq(target.tenant, tenantId), target.available)).limit(1)
|
||||
assignments.push({ kind, id: String(file[kind]), name: entry?.name || (entry ? `#${file[kind]}` : "Nicht mehr verfügbar"), available: Boolean(entry) })
|
||||
}
|
||||
return assignments
|
||||
}
|
||||
export async function validateAssignments(tx: Reader, tenantId: number, assignments: FileAssignments) {
|
||||
const changes: FileAssignments = {}
|
||||
for (const [kind, value] of Object.entries(assignments)) {
|
||||
const target = targetFor(kind)
|
||||
if (value === null) { changes[kind] = null; continue }
|
||||
const id = assignmentId(kind, value)
|
||||
const [row] = await tx.select({ id: target.id }).from(target.table)
|
||||
.where(and(eq(target.id, id), eq(target.tenant, tenantId), target.available)).for("share").limit(1)
|
||||
if (!row) throw invalidAssignment()
|
||||
changes[kind] = id
|
||||
}
|
||||
return changes
|
||||
}
|
||||
@@ -1,12 +1,13 @@
|
||||
import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"
|
||||
import { and, asc, count, desc, eq, ilike, isNull, sql } from "drizzle-orm"
|
||||
import { and, asc, count, desc, eq, ilike, inArray, isNull, sql } from "drizzle-orm"
|
||||
import type { FastifyInstance } from "fastify"
|
||||
import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
|
||||
import { files, filetags, folders, scannerInboxConnections, scannerInboxDocuments, tenants } from "../../db/schema"
|
||||
import { sanitizeFilename } from "../utils/filename"
|
||||
import { s3 } from "../utils/s3"
|
||||
import { secrets } from "../utils/secrets"
|
||||
import { suggestDocumentFiling, validateFilingSuggestion, type FilingSuggestionResult } from "./scanner-inbox-suggestions"
|
||||
import { suggestDocumentFiling, validateFilingSuggestion } from "./scanner-inbox-suggestions"
|
||||
import { assignmentKinds, assignmentOptions, currentAssignments, validateAssignments, type FileAssignments } from "./scanner-inbox-assignments"
|
||||
|
||||
export const SCANNER_UPLOAD_LIMIT = 100 * 1024 * 1024
|
||||
export const SCANNER_DAV_PATH = "/scanner-inbox/"
|
||||
@@ -47,7 +48,6 @@ export function validateScannerFile(filename: string, body: Buffer) {
|
||||
}
|
||||
|
||||
export function createScannerInboxService(db: FastifyInstance["db"], storage: ScannerStorage = objectStorage, analyze = suggestDocumentFiling) {
|
||||
const pendingSuggestions = new Map<string, Promise<FilingSuggestionResult>>()
|
||||
const publicConnection = {
|
||||
username: scannerInboxConnections.username,
|
||||
active: scannerInboxConnections.active,
|
||||
@@ -93,13 +93,19 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
|
||||
isNull(scannerInboxDocuments.filedAt), eq(files.archived, false),
|
||||
search ? ilike(scannerInboxDocuments.remoteFilename, `%${search.replace(/[\\%_]/g, "\\$&")}%`) : undefined)
|
||||
const documents = await db.select({ id: files.id, name: scannerInboxDocuments.remoteFilename,
|
||||
mimeType: files.mimeType, size: files.size, receivedAt: scannerInboxDocuments.receivedAt })
|
||||
mimeType: files.mimeType, size: files.size, receivedAt: scannerInboxDocuments.receivedAt, suggestionStatus: scannerInboxDocuments.suggestionStatus })
|
||||
.from(scannerInboxDocuments).innerJoin(files, eq(files.id, scannerInboxDocuments.fileId))
|
||||
.where(condition).orderBy(desc(scannerInboxDocuments.receivedAt), desc(files.id)).limit(limit).offset(offset)
|
||||
const [total] = await db.select({ value: count() }).from(scannerInboxDocuments)
|
||||
.innerJoin(files, eq(files.id, scannerInboxDocuments.fileId)).where(condition)
|
||||
return { documents, total: total.value }
|
||||
}
|
||||
const openCount = async (tenantId: number) => {
|
||||
const [row] = await db.select({ total: count() }).from(scannerInboxDocuments)
|
||||
.innerJoin(files, eq(files.id, scannerInboxDocuments.fileId))
|
||||
.where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(files.tenant, tenantId), isNull(scannerInboxDocuments.filedAt), eq(files.archived, false)))
|
||||
return row
|
||||
}
|
||||
const filingOptions = async (tenantId: number) => {
|
||||
const [availableFolders, types] = await Promise.all([
|
||||
db.select({ id: folders.id, name: folders.name, parent: folders.parent,
|
||||
@@ -108,33 +114,77 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
|
||||
db.select({ id: filetags.id, name: filetags.name }).from(filetags)
|
||||
.where(and(eq(filetags.tenant, tenantId), eq(filetags.archived, false))).orderBy(asc(filetags.name)),
|
||||
])
|
||||
return { folders: availableFolders, types }
|
||||
return { folders: availableFolders, types, assignmentKinds }
|
||||
}
|
||||
const suggestionDocument = async (tenantId: number, fileId: string) => {
|
||||
const [document] = await db.select({ path: files.path, name: scannerInboxDocuments.remoteFilename, mimeType: files.mimeType, size: files.size })
|
||||
const [document] = await db.select({ path: files.path, name: scannerInboxDocuments.remoteFilename, mimeType: files.mimeType, size: files.size,
|
||||
status: scannerInboxDocuments.suggestionStatus, suggestion: scannerInboxDocuments.suggestion, message: scannerInboxDocuments.suggestionMessage })
|
||||
.from(scannerInboxDocuments).innerJoin(files, eq(files.id, scannerInboxDocuments.fileId))
|
||||
.where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(files.tenant, tenantId), eq(files.id, fileId),
|
||||
isNull(scannerInboxDocuments.filedAt), eq(files.archived, false))).limit(1)
|
||||
if (!document?.path) throw scannerError(404, "Dokument nicht in der Ablage gefunden.")
|
||||
return document
|
||||
}
|
||||
const suggestFiling = async (tenantId: number, fileId: string): Promise<FilingSuggestionResult> => {
|
||||
// Check tenant ownership before reading storage or submitting anything to the AI provider.
|
||||
const getSuggestion = async (tenantId: number, fileId: string) => {
|
||||
const document = await suggestionDocument(tenantId, fileId)
|
||||
const key = `${tenantId}:${fileId}`
|
||||
if (pendingSuggestions.has(key)) return pendingSuggestions.get(key)!
|
||||
if (pendingSuggestions.size >= 4) throw scannerError(429, "Die KI verarbeitet gerade weitere Dokumente. Bitte gleich erneut versuchen.")
|
||||
const operation = (async () => {
|
||||
const result = await analyze(document, await filingOptions(tenantId))
|
||||
await suggestionDocument(tenantId, fileId)
|
||||
const suggestion = validateFilingSuggestion(result, await filingOptions(tenantId))
|
||||
return { suggestion, message: suggestion ? undefined : result.message || "Keine passende Zuordnung verfügbar. Bitte manuell auswählen." }
|
||||
})()
|
||||
pendingSuggestions.set(key, operation)
|
||||
try { return await operation }
|
||||
finally { pendingSuggestions.delete(key) }
|
||||
const suggestion = document.status === "ready"
|
||||
? validateFilingSuggestion({ suggestion: document.suggestion }, await filingOptions(tenantId)) : null
|
||||
return { status: document.status, suggestion, message: document.status === "ready" && !suggestion
|
||||
? document.message || "Keine passende Zuordnung verfügbar. Bitte manuell auswählen." : document.message }
|
||||
}
|
||||
const fileDocument = async (tenantId: number, userId: string, fileId: string, folderId: string | null, typeId: string) => db.transaction(async tx => {
|
||||
const queueSuggestion = async (tenantId: number, fileId: string) => {
|
||||
await suggestionDocument(tenantId, fileId)
|
||||
await db.update(scannerInboxDocuments).set({ suggestionStatus: "pending", suggestion: null, suggestionMessage: null,
|
||||
suggestionClaim: null, suggestionUpdatedAt: new Date() })
|
||||
.where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(scannerInboxDocuments.fileId, fileId),
|
||||
isNull(scannerInboxDocuments.filedAt), inArray(scannerInboxDocuments.suggestionStatus, ["ready", "failed"])))
|
||||
return getSuggestion(tenantId, fileId)
|
||||
}
|
||||
const processNextSuggestion = async () => {
|
||||
const claim = randomUUID()
|
||||
// Claim without holding a transaction open while OCR/the provider runs. Expired claims survive restarts.
|
||||
const claimed = await db.execute<{ fileId: string; tenantId: string }>(sql`
|
||||
UPDATE scanner_inbox_documents AS job
|
||||
SET suggestion_status = 'processing', suggestion_claim = ${claim}, suggestion_updated_at = now()
|
||||
WHERE job.file_id = (
|
||||
SELECT pending.file_id FROM scanner_inbox_documents AS pending
|
||||
JOIN files ON files.id = pending.file_id AND files.tenant = pending.tenant_id
|
||||
JOIN tenants ON tenants.id = pending.tenant_id
|
||||
WHERE pending.filed_at IS NULL AND files.archived = false
|
||||
AND tenants.locked IS DISTINCT FROM 'maintenance_tenant'
|
||||
AND (pending.suggestion_status = 'pending' OR
|
||||
(pending.suggestion_status = 'processing' AND pending.suggestion_updated_at < now() - interval '5 minutes'))
|
||||
ORDER BY pending.received_at, pending.file_id FOR UPDATE OF pending SKIP LOCKED LIMIT 1
|
||||
) RETURNING job.file_id AS "fileId", job.tenant_id AS "tenantId"`)
|
||||
const job = claimed.rows[0]
|
||||
if (!job) return false
|
||||
const tenantId = Number(job.tenantId)
|
||||
const condition = and(eq(scannerInboxDocuments.fileId, job.fileId), eq(scannerInboxDocuments.tenantId, tenantId),
|
||||
eq(scannerInboxDocuments.suggestionClaim, claim), isNull(scannerInboxDocuments.filedAt))
|
||||
try {
|
||||
const document = await suggestionDocument(tenantId, job.fileId)
|
||||
const { path, name, mimeType, size } = document
|
||||
const result = await analyze({ path, name, mimeType, size }, await filingOptions(tenantId))
|
||||
await suggestionDocument(tenantId, job.fileId)
|
||||
const suggestion = validateFilingSuggestion(result, await filingOptions(tenantId))
|
||||
await db.update(scannerInboxDocuments).set({ suggestionStatus: "ready", suggestion,
|
||||
suggestionMessage: suggestion ? null : result.message || "Keine passende Zuordnung gefunden. Bitte manuell auswählen.",
|
||||
suggestionClaim: null, suggestionUpdatedAt: new Date() }).where(condition)
|
||||
} catch (error: any) {
|
||||
const message = error?.statusCode === 503 ? "Die KI-Erkennung ist noch nicht konfiguriert. Bitte manuell zuordnen oder später erneut prüfen."
|
||||
: "Die KI-Vorbereitung ist fehlgeschlagen. Bitte erneut prüfen oder manuell zuordnen."
|
||||
await db.update(scannerInboxDocuments).set({ suggestionStatus: "failed", suggestion: null, suggestionMessage: message,
|
||||
suggestionClaim: null, suggestionUpdatedAt: new Date() }).where(condition)
|
||||
}
|
||||
return true
|
||||
}
|
||||
const getAssignments = async (tenantId: number, fileId: string) => {
|
||||
await suggestionDocument(tenantId, fileId)
|
||||
const [file] = await db.select().from(files).where(and(eq(files.id, fileId), eq(files.tenant, tenantId), eq(files.archived, false))).limit(1)
|
||||
if (!file) throw scannerError(404, "Dokument nicht gefunden.")
|
||||
return currentAssignments(db, tenantId, file)
|
||||
}
|
||||
const fileDocument = async (tenantId: number, userId: string, fileId: string, folderId: string | null, typeId: string, assignments: FileAssignments = {}) => db.transaction(async tx => {
|
||||
const [tenant] = await tx.select({ locked: tenants.locked }).from(tenants)
|
||||
.where(eq(tenants.id, tenantId)).for("share")
|
||||
if (!tenant) throw scannerError(404, "Mandant nicht gefunden.")
|
||||
@@ -157,11 +207,12 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
|
||||
const [type] = await tx.select({ id: filetags.id }).from(filetags)
|
||||
.where(and(eq(filetags.id, typeId), eq(filetags.tenant, tenantId), eq(filetags.archived, false))).for("share")
|
||||
if (!type) throw scannerError(400, "Der gewählte Dokumenttyp ist nicht verfügbar.")
|
||||
const assignmentChanges = await validateAssignments(tx, tenantId, assignments)
|
||||
const now = new Date()
|
||||
await tx.update(files).set({ folder: folderId, type: typeId, updatedAt: now, updatedBy: userId })
|
||||
await tx.update(files).set({ ...assignmentChanges, folder: folderId, type: typeId, updatedAt: now, updatedBy: userId })
|
||||
.where(and(eq(files.id, fileId), eq(files.tenant, tenantId)))
|
||||
// Keep the receipt so a scanner retry cannot recreate a document after filing.
|
||||
await tx.update(scannerInboxDocuments).set({ filedAt: now })
|
||||
await tx.update(scannerInboxDocuments).set({ filedAt: now, suggestionClaim: null })
|
||||
.where(and(eq(scannerInboxDocuments.fileId, fileId), eq(scannerInboxDocuments.tenantId, tenantId)))
|
||||
return { id: fileId, folder: folderId, type: typeId }
|
||||
})
|
||||
@@ -225,7 +276,8 @@ export function createScannerInboxService(db: FastifyInstance["db"], storage: Sc
|
||||
throw error
|
||||
}
|
||||
}
|
||||
return { connection, credentials, disable, authenticate, find, list, filingOptions, suggestFiling, fileDocument, probe, accept }
|
||||
return { connection, credentials, disable, authenticate, find, list, openCount, filingOptions, getSuggestion, queueSuggestion, processNextSuggestion,
|
||||
getAssignments, assignmentOptions: (tenantId: number, kind: string, search: string) => assignmentOptions(db, tenantId, kind, search), fileDocument, probe, accept }
|
||||
}
|
||||
|
||||
export type ScannerInboxService = ReturnType<typeof createScannerInboxService>
|
||||
|
||||
27
backend/src/modules/scanner-inbox.worker.ts
Normal file
27
backend/src/modules/scanner-inbox.worker.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import type { FastifyInstance } from "fastify"
|
||||
import { createScannerInboxService, type ScannerInboxService } from "./scanner-inbox.service"
|
||||
|
||||
export function startScannerInboxWorker(server: FastifyInstance, service: Pick<ScannerInboxService, "processNextSuggestion"> = createScannerInboxService(server.db)) {
|
||||
let stopped = false
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let running: Promise<void> | undefined
|
||||
const run = async () => {
|
||||
try {
|
||||
// One analysis per process; database claims coordinate multiple backend instances.
|
||||
while (!stopped && await service.processNextSuggestion()) { /* drain the durable queue */ }
|
||||
} catch {
|
||||
server.log.error("KI-Vorbereitung der Dokumentenablage unterbrochen; erneuter Versuch folgt.")
|
||||
} finally {
|
||||
if (!stopped) {
|
||||
timer = setTimeout(() => { running = run() }, 2000)
|
||||
timer.unref()
|
||||
}
|
||||
}
|
||||
}
|
||||
server.addHook("onReady", async () => { running = run() })
|
||||
server.addHook("onClose", async () => {
|
||||
stopped = true
|
||||
clearTimeout(timer)
|
||||
await running
|
||||
})
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { and, eq } from "drizzle-orm"
|
||||
import type { FastifyInstance } from "fastify"
|
||||
import { authProfiles } from "../../db/schema"
|
||||
import { createScannerInboxService, SCANNER_DAV_PATH, SCANNER_UPLOAD_LIMIT, type ScannerInboxService } from "../modules/scanner-inbox.service"
|
||||
import { assignmentTargets, type FileAssignments } from "../modules/scanner-inbox-assignments"
|
||||
|
||||
export default async function documentInboxRoutes(server: FastifyInstance, options: { service?: ScannerInboxService }) {
|
||||
const service = options.service || createScannerInboxService(server.db)
|
||||
@@ -25,18 +26,32 @@ export default async function documentInboxRoutes(server: FastifyInstance, optio
|
||||
return service.list(req.user.tenant_id!, limit, offset, query.search?.trim() || "")
|
||||
})
|
||||
server.get("/document-inbox/filing-options", async req => service.filingOptions(req.user.tenant_id!))
|
||||
server.get("/document-inbox/count", async req => service.openCount(req.user.tenant_id!))
|
||||
server.get<{ Querystring: { kind: string; search?: string } }>("/document-inbox/assignment-options", {
|
||||
schema: { querystring: { type: "object", required: ["kind"], properties: {
|
||||
kind: { type: "string", enum: Object.keys(assignmentTargets) }, search: { type: "string", maxLength: 200 },
|
||||
} } },
|
||||
}, async req => service.assignmentOptions(req.user.tenant_id!, req.query.kind, req.query.search?.trim() || ""))
|
||||
server.get<{ Params: { id: string } }>("/document-inbox/:id/assignments", {
|
||||
schema: { params: { type: "object", required: ["id"], properties: { id: { type: "string", format: "uuid" } } } },
|
||||
}, async req => service.getAssignments(req.user.tenant_id!, req.params.id))
|
||||
server.get<{ Params: { id: string } }>("/document-inbox/:id/suggestion", {
|
||||
schema: { params: { type: "object", required: ["id"], properties: { id: { type: "string", format: "uuid" } } } },
|
||||
}, async req => service.getSuggestion(req.user.tenant_id!, req.params.id))
|
||||
server.post<{ Params: { id: string } }>("/document-inbox/:id/suggestion", {
|
||||
schema: { params: { type: "object", required: ["id"], properties: { id: { type: "string", format: "uuid" } } } },
|
||||
}, async req => service.suggestFiling(req.user.tenant_id!, req.params.id))
|
||||
server.post<{ Params: { id: string }; Body: { folder: string | null; type: string } }>("/document-inbox/:id/file", {
|
||||
}, async (req, reply) => reply.code(202).send(await service.queueSuggestion(req.user.tenant_id!, req.params.id)))
|
||||
server.post<{ Params: { id: string }; Body: { folder: string | null; type: string; assignments?: FileAssignments } }>("/document-inbox/:id/file", {
|
||||
schema: {
|
||||
params: { type: "object", required: ["id"], properties: { id: { type: "string", format: "uuid" } } },
|
||||
body: { type: "object", required: ["folder", "type"], additionalProperties: false, properties: {
|
||||
folder: { anyOf: [{ type: "string", format: "uuid" }, { type: "null" }] },
|
||||
type: { type: "string", format: "uuid" },
|
||||
assignments: { type: "object", additionalProperties: false, properties: Object.fromEntries(Object.keys(assignmentTargets).map(kind =>
|
||||
[kind, { anyOf: [{ type: "null" }, { type: "integer", minimum: 1, maximum: Number.MAX_SAFE_INTEGER }, { type: "string", minLength: 1, maxLength: 36 }] }])) },
|
||||
} },
|
||||
},
|
||||
}, async req => service.fileDocument(req.user.tenant_id!, req.user.user_id, req.params.id, req.body.folder, req.body.type))
|
||||
}, async req => service.fileDocument(req.user.tenant_id!, req.user.user_id, req.params.id, req.body.folder, req.body.type, req.body.assignments))
|
||||
server.get("/document-inbox/scanner", async req => ({
|
||||
connection: await service.connection(req.user.tenant_id!), path: SCANNER_DAV_PATH, maxFileSize: SCANNER_UPLOAD_LIMIT,
|
||||
}))
|
||||
|
||||
@@ -193,10 +193,14 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
CREATE TABLE filetags (id uuid PRIMARY KEY, tenant bigint, name text, archived boolean DEFAULT false);
|
||||
CREATE TABLE folders (id uuid PRIMARY KEY, tenant bigint, name text, parent uuid, "standardFiletype" uuid,
|
||||
"standardFiletypeIsOptional" boolean DEFAULT true, archived boolean DEFAULT false);
|
||||
CREATE TABLE auth_profiles (tenant_id bigint, user_id uuid, customer_for_portal bigint);`)
|
||||
CREATE TABLE projects (id bigint PRIMARY KEY, tenant bigint, name text, archived boolean DEFAULT false);
|
||||
CREATE TABLE customers (id bigint PRIMARY KEY, tenant bigint, name text, archived boolean DEFAULT false);
|
||||
CREATE TABLE checks (id uuid PRIMARY KEY, tenant bigint, name text, archived boolean DEFAULT false);
|
||||
CREATE TABLE auth_profiles (id uuid DEFAULT gen_random_uuid(), tenant_id bigint, user_id uuid, customer_for_portal bigint,
|
||||
full_name text, active boolean DEFAULT true);`)
|
||||
const migrationRoot = path.resolve(__dirname, "../db/migrations")
|
||||
const journal = JSON.parse(await readFile(path.join(migrationRoot, "meta/_journal.json"), "utf8"))
|
||||
const entries = ["0073_scanner_document_inbox", "0074_scanner_inbox_filing"].map(tag => journal.entries.find((item: any) => item.tag === tag))
|
||||
const entries = ["0073_scanner_document_inbox", "0074_scanner_inbox_filing", "0077_scanner_inbox_preparation"].map(tag => journal.entries.find((item: any) => item.tag === tag))
|
||||
await mkdir(path.join(tempMigrations, "meta"))
|
||||
for (const entry of entries) {
|
||||
assert.ok(entry)
|
||||
@@ -213,14 +217,17 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
const analyses: any[] = []
|
||||
let analysisGate: Promise<void> | undefined
|
||||
let analysisStarted: (() => void) | undefined
|
||||
let analysisFailure = false
|
||||
const service = createScannerInboxService(db, {
|
||||
async put(key, body) { stored.set(key, Buffer.from(body)); if (storageFailure) throw new Error("Simulated object store failure") },
|
||||
async remove(key) { stored.delete(key) },
|
||||
}, async (document, candidates) => {
|
||||
analyses.push({ document, candidates })
|
||||
const result = aiResult
|
||||
analysisStarted?.()
|
||||
await analysisGate
|
||||
return aiResult
|
||||
if (analysisFailure) throw new Error("Simulated AI failure")
|
||||
return result
|
||||
})
|
||||
const first = await service.credentials(1)
|
||||
const second = await service.credentials(2)
|
||||
@@ -265,6 +272,8 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
assert.equal(row.tenant, "1")
|
||||
assert.equal(row.mimeType, "application/pdf")
|
||||
assert.equal(row.extracted_text, null)
|
||||
assert.equal((await pool.query("SELECT suggestion_status FROM scanner_inbox_documents WHERE file_id=$1", [row.id])).rows[0].suggestion_status, "pending")
|
||||
assert.equal(analyses.length, 0, "the upload only queues preparation; it does not wait for OCR/AI")
|
||||
assert.deepEqual(stored.get(row.path), pdf)
|
||||
assert.equal((await PDFDocument.load(stored.get(row.path)!)).getPageCount(), 2)
|
||||
const listing = await app.inject({ method: "GET", url: "/api/document-inbox" })
|
||||
@@ -332,17 +341,21 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
[targetFolder, targetType, foreignFolder, archivedFolder])
|
||||
const fileInto = (id: string, folder: string | null = targetFolder, type: string = targetType, headers = {}) =>
|
||||
app.inject({ method: "POST", url: `/api/document-inbox/${id}/file`, payload: { folder, type }, headers })
|
||||
const suggest = (id: string, headers = {}) => app.inject({ method: "POST", url: `/api/document-inbox/${id}/suggestion`, headers })
|
||||
const suggest = (id: string, headers = {}) => app.inject({ method: "GET", url: `/api/document-inbox/${id}/suggestion`, headers })
|
||||
const queue = (id: string, headers = {}) => app.inject({ method: "POST", url: `/api/document-inbox/${id}/suggestion`, headers })
|
||||
await t.test("AI analysis is tenant scoped, validates destinations and never files a document", async () => {
|
||||
const id = (await service.list(1, 50, 0, "scan.pdf")).documents[0].id
|
||||
const before = (await pool.query("SELECT folder, type, extracted_text FROM files WHERE id=$1", [id])).rows[0]
|
||||
aiResult = { suggestion: { folderId: targetFolder, typeId: targetType, confidence: "high", reason: "Rechnung erkannt." } }
|
||||
assert.equal((await suggest(id, { "x-test-tenant": "2" })).statusCode, 404)
|
||||
assert.equal((await suggest("invalid")).statusCode, 400)
|
||||
await pool.query("INSERT INTO auth_profiles VALUES (1, $1, 123)", [userId])
|
||||
await pool.query("INSERT INTO auth_profiles (tenant_id, user_id, customer_for_portal) VALUES (1, $1, 123)", [userId])
|
||||
assert.equal((await suggest(id)).statusCode, 403)
|
||||
await pool.query("DELETE FROM auth_profiles")
|
||||
assert.equal(analyses.length, 0)
|
||||
assert.equal((await suggest(id)).json().status, "pending")
|
||||
assert.equal(analyses.length, 0, "reading the preview never starts analysis")
|
||||
assert.equal(await service.processNextSuggestion(), true)
|
||||
const result = await suggest(id, { "x-test-admin": "false" })
|
||||
assert.equal(result.statusCode, 200, result.body)
|
||||
assert.deepEqual(result.json().suggestion, aiResult.suggestion)
|
||||
@@ -352,6 +365,8 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
assert.deepEqual((await pool.query("SELECT folder, type, extracted_text FROM files WHERE id=$1", [id])).rows[0], before)
|
||||
assert.equal((await service.list(1, 50, 0, "scan.pdf")).total, 1)
|
||||
aiResult = { suggestion: { ...aiResult.suggestion!, folderId: foreignFolder } }
|
||||
assert.equal((await queue(id)).statusCode, 202)
|
||||
await service.processNextSuggestion()
|
||||
assert.equal((await suggest(id)).json().suggestion, null)
|
||||
})
|
||||
await t.test("AI results are checked again if destination availability changes during analysis", async () => {
|
||||
@@ -360,15 +375,93 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
let finish!: () => void
|
||||
analysisGate = new Promise(resolve => { finish = resolve })
|
||||
const started = new Promise<void>(resolve => { analysisStarted = resolve })
|
||||
const pending = suggest(id).then(response => response)
|
||||
await queue(id)
|
||||
const pending = service.processNextSuggestion()
|
||||
await started
|
||||
await pool.query("UPDATE folders SET archived=true WHERE id=$1", [targetFolder])
|
||||
finish()
|
||||
assert.equal((await pending).json().suggestion, null)
|
||||
await pending
|
||||
assert.equal((await suggest(id)).json().suggestion, null)
|
||||
await pool.query("UPDATE folders SET archived=false WHERE id=$1", [targetFolder])
|
||||
analysisGate = undefined
|
||||
analysisStarted = undefined
|
||||
})
|
||||
await t.test("prepared results persist, failed jobs wait for explicit retry and expired claims resume", async () => {
|
||||
const id = (await service.list(1, 50, 0, "scan.pdf")).documents[0].id
|
||||
const otherService = createScannerInboxService(db, undefined, async () => aiResult)
|
||||
const before = analyses.length
|
||||
assert.equal((await otherService.getSuggestion(1, id)).status, "ready")
|
||||
assert.equal(analyses.length, before)
|
||||
await queue(id)
|
||||
analysisFailure = true
|
||||
await service.processNextSuggestion()
|
||||
analysisFailure = false
|
||||
assert.equal((await suggest(id)).json().status, "failed")
|
||||
assert.equal((await suggest(id)).json().suggestion, null)
|
||||
const failedCount = analyses.length
|
||||
await suggest(id)
|
||||
assert.equal(analyses.length, failedCount)
|
||||
await queue(id)
|
||||
await pool.query("UPDATE scanner_inbox_documents SET suggestion_status='processing', suggestion_claim=$2, suggestion_updated_at=now()-interval '6 minutes' WHERE file_id=$1", [id, randomUUID()])
|
||||
aiResult = { suggestion: { folderId: targetFolder, typeId: targetType, confidence: "high", reason: "Vorbereitet nach Neustart." } }
|
||||
await otherService.processNextSuggestion()
|
||||
assert.equal((await suggest(id)).json().suggestion.reason, "Vorbereitet nach Neustart.")
|
||||
})
|
||||
await t.test("an expired worker cannot overwrite a newer completed suggestion", async () => {
|
||||
const id = (await service.list(1, 50, 0, "scan.pdf")).documents[0].id
|
||||
await queue(id)
|
||||
aiResult = { suggestion: { folderId: targetFolder, typeId: targetType, confidence: "high", reason: "Alter Vorschlag." } }
|
||||
let finish!: () => void
|
||||
analysisGate = new Promise(resolve => { finish = resolve })
|
||||
const started = new Promise<void>(resolve => { analysisStarted = resolve })
|
||||
const pending = service.processNextSuggestion()
|
||||
await started
|
||||
await pool.query("UPDATE scanner_inbox_documents SET suggestion_updated_at=now()-interval '6 minutes' WHERE file_id=$1", [id])
|
||||
const otherService = createScannerInboxService(db, undefined, async () => ({ suggestion: {
|
||||
folderId: null, typeId: alternativeType, confidence: "medium", reason: "Neuer Vorschlag.",
|
||||
} }))
|
||||
await otherService.processNextSuggestion()
|
||||
finish()
|
||||
await pending
|
||||
assert.equal((await suggest(id)).json().suggestion.reason, "Neuer Vorschlag.")
|
||||
analysisGate = undefined
|
||||
analysisStarted = undefined
|
||||
})
|
||||
await t.test("entity choices and filing reject foreign, archived and wrong-type IDs atomically", async () => {
|
||||
await pool.query("INSERT INTO projects VALUES (11, 1, 'Umbau', false), (12, 2, 'Fremdes Projekt', false), (13, 1, 'Archiv', true)")
|
||||
await pool.query("INSERT INTO customers VALUES (21, 1, 'Musterkunde', false), (22, 2, 'Fremder Kunde', false)")
|
||||
const checkId = randomUUID()
|
||||
const profileId = randomUUID()
|
||||
await pool.query("INSERT INTO checks VALUES ($1, 1, 'Prüfung', false)", [checkId])
|
||||
await pool.query("INSERT INTO auth_profiles (id, tenant_id, full_name) VALUES ($1, 1, 'Test Mitarbeiter')", [profileId])
|
||||
const choices = await app.inject({ method: "GET", url: "/api/document-inbox/assignment-options?kind=project&search=Umbau" })
|
||||
assert.deepEqual(choices.json(), [{ id: "11", name: "Umbau" }])
|
||||
assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox/assignment-options?kind=unknown" })).statusCode, 400)
|
||||
const id = (await service.list(1, 50, 0, "scan.pdf")).documents[0].id
|
||||
for (const assignments of [{ project: "12" }, { project: "13" }, { project: randomUUID() }, { project: "11", customer: "22" }, { check: 11 }, { authProfile: randomUUID() }]) {
|
||||
const res = await app.inject({ method: "POST", url: `/api/document-inbox/${id}/file`, payload: { folder: targetFolder, type: targetType, assignments } })
|
||||
assert.equal(res.statusCode, 400, res.body)
|
||||
}
|
||||
assert.deepEqual((await pool.query("SELECT folder, type, project, customer FROM files WHERE id=$1", [id])).rows[0], { folder: null, type: null, project: null, customer: null })
|
||||
await pool.query("UPDATE files SET project=11 WHERE id=$1", [id])
|
||||
assert.deepEqual((await app.inject({ method: "GET", url: `/api/document-inbox/${id}/assignments` })).json(), [{ kind: "project", id: "11", name: "Umbau", available: true }])
|
||||
assert.equal((await app.inject({ method: "GET", url: `/api/document-inbox/${id}/assignments`, headers: { "x-test-tenant": "2" } })).statusCode, 404)
|
||||
await service.fileDocument(1, userId, id, targetFolder, targetType, { project: null, customer: "21", check: checkId, authProfile: profileId })
|
||||
assert.deepEqual((await pool.query('SELECT project, customer, "check", auth_profile FROM files WHERE id=$1', [id])).rows[0], { project: null, customer: '21', check: checkId, auth_profile: profileId })
|
||||
// Restore the fixture so existing retry/filing tests can exercise this same original document.
|
||||
await pool.query('UPDATE files SET folder=null, type=null, customer=null, "check"=null, auth_profile=null WHERE id=$1', [id])
|
||||
await pool.query("UPDATE scanner_inbox_documents SET filed_at=null WHERE file_id=$1", [id])
|
||||
})
|
||||
await t.test("menu count includes only this tenant's open, unarchived documents", async () => {
|
||||
const res = await app.inject({ method: "GET", url: "/api/document-inbox/count" })
|
||||
assert.equal(res.json().total, (await service.list(1, 50, 0, "")).total)
|
||||
const other = await app.inject({ method: "GET", url: "/api/document-inbox/count", headers: { "x-test-tenant": "2" } })
|
||||
assert.equal(other.json().total, 1)
|
||||
const id = (await service.list(1, 50, 0, "failed.pdf")).documents[0].id
|
||||
await pool.query("UPDATE files SET archived=true WHERE id=$1", [id])
|
||||
assert.equal((await service.openCount(1)).total, res.json().total - 1)
|
||||
await pool.query("UPDATE files SET archived=false WHERE id=$1", [id])
|
||||
})
|
||||
await t.test("filing options and destinations exclude other tenants and archived entries", async () => {
|
||||
const options = await app.inject({ method: "GET", url: "/api/document-inbox/filing-options" })
|
||||
assert.equal(options.statusCode, 200)
|
||||
@@ -388,7 +481,7 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
await pool.query("UPDATE tenants SET locked='maintenance_tenant' WHERE id=1")
|
||||
assert.equal((await fileInto(id)).statusCode, 423)
|
||||
await pool.query("UPDATE tenants SET locked=null WHERE id=1")
|
||||
await pool.query("INSERT INTO auth_profiles VALUES (1, $1, 123)", [userId])
|
||||
await pool.query("INSERT INTO auth_profiles (tenant_id, user_id, customer_for_portal) VALUES (1, $1, 123)", [userId])
|
||||
assert.equal((await fileInto(id)).statusCode, 403)
|
||||
assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox/filing-options" })).statusCode, 403)
|
||||
await pool.query("DELETE FROM auth_profiles")
|
||||
@@ -444,7 +537,7 @@ test("scanner inbox integration: migration, persistence, isolation, retries and
|
||||
assert.equal((await app.inject({ method, url, headers: { "x-test-admin": "false" } })).statusCode, 403)
|
||||
}
|
||||
assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox", headers: { "x-test-admin": "false" } })).statusCode, 200)
|
||||
await pool.query("INSERT INTO auth_profiles VALUES (1, $1, 123)", [userId])
|
||||
await pool.query("INSERT INTO auth_profiles (tenant_id, user_id, customer_for_portal) VALUES (1, $1, 123)", [userId])
|
||||
assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox" })).statusCode, 403)
|
||||
await pool.query("DELETE FROM auth_profiles")
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user