diff --git a/backend/db/migrations/0073_scanner_document_inbox.sql b/backend/db/migrations/0073_scanner_document_inbox.sql new file mode 100644 index 00000000..b1311e70 --- /dev/null +++ b/backend/db/migrations/0073_scanner_document_inbox.sql @@ -0,0 +1,20 @@ +CREATE TABLE "scanner_inbox_connections" ( + "tenant_id" bigint PRIMARY KEY REFERENCES "tenants"("id") ON DELETE CASCADE, + "username" text NOT NULL UNIQUE, + "password_hash" text NOT NULL, + "active" boolean NOT NULL DEFAULT true, + "created_at" timestamp with time zone NOT NULL DEFAULT now(), + "updated_at" timestamp with time zone +); +--> statement-breakpoint +CREATE TABLE "scanner_inbox_documents" ( + "file_id" uuid PRIMARY KEY REFERENCES "files"("id") ON DELETE CASCADE, + "tenant_id" bigint NOT NULL REFERENCES "tenants"("id") ON DELETE CASCADE, + "remote_filename" text NOT NULL, + "checksum" text NOT NULL, + "received_at" timestamp with time zone NOT NULL DEFAULT now() +); +--> statement-breakpoint +CREATE UNIQUE INDEX "scanner_inbox_documents_filename_key" ON "scanner_inbox_documents" ("tenant_id", "remote_filename"); +--> statement-breakpoint +CREATE INDEX "scanner_inbox_documents_received_idx" ON "scanner_inbox_documents" ("tenant_id", "received_at"); diff --git a/backend/db/migrations/meta/_journal.json b/backend/db/migrations/meta/_journal.json index 3e3ce195..ea8ac0fe 100644 --- a/backend/db/migrations/meta/_journal.json +++ b/backend/db/migrations/meta/_journal.json @@ -491,6 +491,13 @@ "when": 1790121600001, "tag": "0072_restore_instance_scan_agents", "breakpoints": true + }, + { + "idx": 70, + "version": "7", + "when": 1790640000000, + "tag": "0073_scanner_document_inbox", + "breakpoints": true } ] } diff --git a/backend/db/schema/index.ts b/backend/db/schema/index.ts index b37bb941..e7acf454 100644 --- a/backend/db/schema/index.ts +++ b/backend/db/schema/index.ts @@ -100,3 +100,4 @@ export * from "./serialtypes" export * from "./serialexecutions" export * from "./public_links" export * from "./wikipages" +export * from "./scanner_inbox" diff --git a/backend/db/schema/scanner_inbox.ts b/backend/db/schema/scanner_inbox.ts new file mode 100644 index 00000000..f7bededa --- /dev/null +++ b/backend/db/schema/scanner_inbox.ts @@ -0,0 +1,23 @@ +import { bigint, boolean, index, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core" +import { tenants } from "./tenants" +import { files } from "./files" + +export const scannerInboxConnections = pgTable("scanner_inbox_connections", { + tenantId: bigint("tenant_id", { mode: "number" }).primaryKey().references(() => tenants.id, { onDelete: "cascade" }), + username: text("username").notNull().unique(), + passwordHash: text("password_hash").notNull(), + active: boolean("active").notNull().default(true), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }), +}) + +export const scannerInboxDocuments = pgTable("scanner_inbox_documents", { + fileId: uuid("file_id").primaryKey().references(() => files.id, { onDelete: "cascade" }), + tenantId: bigint("tenant_id", { mode: "number" }).notNull().references(() => tenants.id, { onDelete: "cascade" }), + remoteFilename: text("remote_filename").notNull(), + checksum: text("checksum").notNull(), + receivedAt: timestamp("received_at", { withTimezone: true }).notNull().defaultNow(), +}, table => ({ + filenameKey: uniqueIndex("scanner_inbox_documents_filename_key").on(table.tenantId, table.remoteFilename), + receivedIdx: index("scanner_inbox_documents_received_idx").on(table.tenantId, table.receivedAt), +})) diff --git a/backend/src/index.ts b/backend/src/index.ts index b1828a9d..342feebe 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -37,6 +37,8 @@ import telephonyRoutes from "./routes/telephony"; import instanceAgentRoutes from "./routes/instanceAgents"; import instanceAgentGatewayRoutes from "./routes/instanceAgentGateway"; import documentImportRoutes from "./routes/documentImports"; +import scannerInboxGateway from "./routes/scannerInboxGateway"; +import documentInboxRoutes from "./routes/documentInbox"; //Public Links import publiclinksNonAuthenticatedRoutes from "./routes/publiclinks/publiclinks-non-authenticated"; @@ -134,6 +136,8 @@ async function main() { await agentApp.register(instanceAgentGatewayRoutes) },{prefix: "/instance-agent"}) + await app.register(scannerInboxGateway, { prefix: "/scanner-inbox" }); + await app.register(corsPlugin); //Geschützte Routes @@ -167,6 +171,7 @@ async function main() { await subApp.register(telephonyRoutes); await subApp.register(instanceAgentRoutes); await subApp.register(documentImportRoutes); + await subApp.register(documentInboxRoutes); },{prefix: "/api"}) diff --git a/backend/src/modules/scanner-inbox.service.ts b/backend/src/modules/scanner-inbox.service.ts new file mode 100644 index 00000000..566a1a29 --- /dev/null +++ b/backend/src/modules/scanner-inbox.service.ts @@ -0,0 +1,142 @@ +import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto" +import { and, count, desc, eq, ilike, sql } from "drizzle-orm" +import type { FastifyInstance } from "fastify" +import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3" +import { files, scannerInboxConnections, scannerInboxDocuments, tenants } from "../../db/schema" +import { sanitizeFilename } from "../utils/filename" +import { s3 } from "../utils/s3" +import { secrets } from "../utils/secrets" + +export const SCANNER_UPLOAD_LIMIT = 100 * 1024 * 1024 +export const SCANNER_DAV_PATH = "/scanner-inbox/" +export const scannerError = (statusCode: number, message: string) => Object.assign(new Error(message), { statusCode }) +const digest = (value: string | Buffer) => createHash("sha256").update(value).digest("hex") + +export type ScannerIdentity = { tenantId: number; username: string; passwordHash: string } +export interface ScannerStorage { + put(key: string, body: Buffer, contentType: string): Promise + remove(key: string): Promise +} + +const objectStorage: ScannerStorage = { + async put(key, body, contentType) { + await s3.send(new PutObjectCommand({ Bucket: secrets.S3_BUCKET, Key: key, Body: body, ContentType: contentType, ContentLength: body.length })) + }, + async remove(key) { + await s3.send(new DeleteObjectCommand({ Bucket: secrets.S3_BUCKET, Key: key })) + }, +} + +export function validateScannerFile(filename: string, body: Buffer) { + if (!filename || filename.length > 200 || /[\\/\u0000-\u001f\u007f]/.test(filename) || filename.startsWith(".")) { + throw scannerError(400, "Bitte einen einzelnen Dateinamen ohne Unterordner verwenden.") + } + if (!body.length) throw scannerError(400, "Die Datei ist leer.") + if (body.length > SCANNER_UPLOAD_LIMIT) throw scannerError(413, "Die Datei überschreitet 100 MB.") + const extension = filename.split(".").pop()?.toLowerCase() + if (extension === "pdf" && body.subarray(0, 5).toString() === "%PDF-" && body.subarray(-1024).includes(Buffer.from("%%EOF"))) return "application/pdf" + if (["jpg", "jpeg"].includes(extension) && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff) return "image/jpeg" + if (extension === "png" && body.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return "image/png" + if (["tif", "tiff"].includes(extension) && ["49492a00", "4d4d002a"].includes(body.subarray(0, 4).toString("hex"))) return "image/tiff" + throw scannerError(415, "Bitte eine vollständige PDF-, JPEG-, PNG- oder TIFF-Datei übertragen.") +} + +export function createScannerInboxService(db: FastifyInstance["db"], storage: ScannerStorage = objectStorage) { + const publicConnection = { + username: scannerInboxConnections.username, + active: scannerInboxConnections.active, + createdAt: scannerInboxConnections.createdAt, + } + const connection = async (tenantId: number) => { + const [row] = await db.select(publicConnection).from(scannerInboxConnections) + .where(eq(scannerInboxConnections.tenantId, tenantId)).limit(1) + return row || null + } + const credentials = async (tenantId: number) => { + // Epson ES-580W destination passwords are limited to 20 characters (120 random bits). + const password = randomBytes(15).toString("base64url") + const [row] = await db.insert(scannerInboxConnections).values({ + tenantId, username: `scan_${randomBytes(8).toString("hex")}`, passwordHash: digest(password), + }).onConflictDoUpdate({ + target: scannerInboxConnections.tenantId, + set: { passwordHash: digest(password), active: true, updatedAt: new Date() }, + }).returning(publicConnection) + return { ...row, password } + } + const disable = async (tenantId: number) => { + await db.update(scannerInboxConnections).set({ active: false, updatedAt: new Date() }) + .where(eq(scannerInboxConnections.tenantId, tenantId)) + } + const authenticate = async (username: string, password: string): Promise => { + const [row] = await db.select().from(scannerInboxConnections).where(and( + eq(scannerInboxConnections.username, username), eq(scannerInboxConnections.active, true), + )).limit(1) + const expected = row?.passwordHash || "0".repeat(64) + const valid = timingSafeEqual(Buffer.from(digest(password)), Buffer.from(expected)) + return row && valid ? { tenantId: row.tenantId, username: row.username, passwordHash: row.passwordHash } : null + } + const find = async (tenantId: number, filename: string) => { + const [row] = await db.select({ size: files.size, mimeType: files.mimeType, checksum: scannerInboxDocuments.checksum, + receivedAt: scannerInboxDocuments.receivedAt }).from(scannerInboxDocuments) + .innerJoin(files, and(eq(files.id, scannerInboxDocuments.fileId), eq(files.tenant, scannerInboxDocuments.tenantId))) + .where(and(eq(scannerInboxDocuments.tenantId, tenantId), eq(scannerInboxDocuments.remoteFilename, filename))).limit(1) + return row || null + } + const list = async (tenantId: number, limit: number, offset: number, search: string) => { + const condition = and(eq(scannerInboxDocuments.tenantId, tenantId), eq(files.tenant, tenantId), + search ? ilike(scannerInboxDocuments.remoteFilename, `%${search.replace(/[\\%_]/g, "\\$&")}%`) : undefined) + const documents = await db.select({ id: files.id, name: scannerInboxDocuments.remoteFilename, + mimeType: files.mimeType, size: files.size, receivedAt: scannerInboxDocuments.receivedAt }) + .from(scannerInboxDocuments).innerJoin(files, eq(files.id, scannerInboxDocuments.fileId)) + .where(condition).orderBy(desc(scannerInboxDocuments.receivedAt), desc(files.id)).limit(limit).offset(offset) + const [total] = await db.select({ value: count() }).from(scannerInboxDocuments) + .innerJoin(files, eq(files.id, scannerInboxDocuments.fileId)).where(condition) + return { documents, total: total.value } + } + const accept = async (identity: ScannerIdentity, filename: string, body: Buffer, onlyIfNew = false) => { + const contentType = validateScannerFile(filename, body) + const checksum = digest(body) + let uploadedKey: string | undefined + try { + return await db.transaction(async tx => { + // Coordinate with tenant maintenance and credential revocation during the upload. + const [tenant] = await tx.select({ locked: tenants.locked }).from(tenants) + .where(eq(tenants.id, identity.tenantId)).for("share") + if (!tenant) throw scannerError(401, "Scanner-Zugang ungültig.") + if (tenant.locked === "maintenance_tenant") throw scannerError(423, "Der Mandant wird gerade gewartet.") + const [active] = await tx.select({ tenantId: scannerInboxConnections.tenantId }).from(scannerInboxConnections) + .where(and(eq(scannerInboxConnections.tenantId, identity.tenantId), eq(scannerInboxConnections.username, identity.username), + eq(scannerInboxConnections.passwordHash, identity.passwordHash), eq(scannerInboxConnections.active, true))).for("share") + if (!active) throw scannerError(401, "Scanner-Zugang ungültig.") + + // Serialise retries across processes; a lost HTTP response must not duplicate a document. + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended(${`${identity.tenantId}:${filename}`}, 0))`) + const [existing] = await tx.select().from(scannerInboxDocuments).where(and( + eq(scannerInboxDocuments.tenantId, identity.tenantId), eq(scannerInboxDocuments.remoteFilename, filename), + )).limit(1) + if (existing) { + if (onlyIfNew) throw scannerError(412, "Der Dateiname ist bereits vorhanden.") + if (existing.checksum !== checksum) throw scannerError(409, "Der Dateiname ist bereits vergeben. Bitte einen eindeutigen Dateinamen verwenden.") + return { created: false, checksum } + } + const id = randomUUID() + const key = `${identity.tenantId}/filesbyid/${id}/${sanitizeFilename(filename)}` + uploadedKey = key + await storage.put(key, body, contentType) + // Bypass saveFile's text extraction: the inbox only stores the original document. + await tx.insert(files).values({ id, tenant: identity.tenantId, name: filename, path: key, mimeType: contentType, size: body.length }) + await tx.insert(scannerInboxDocuments).values({ fileId: id, tenantId: identity.tenantId, remoteFilename: filename, checksum }) + return { created: true, checksum } + }) + } catch (error) { + if (uploadedKey) { + try { await storage.remove(uploadedKey) } + catch (cleanupError) { console.error("Scanner-Upload konnte nicht bereinigt werden", { key: uploadedKey, cleanupError }) } + } + throw error + } + } + return { connection, credentials, disable, authenticate, find, list, accept } +} + +export type ScannerInboxService = ReturnType diff --git a/backend/src/routes/documentInbox.ts b/backend/src/routes/documentInbox.ts new file mode 100644 index 00000000..ac6d79e2 --- /dev/null +++ b/backend/src/routes/documentInbox.ts @@ -0,0 +1,37 @@ +import { and, eq } from "drizzle-orm" +import type { FastifyInstance } from "fastify" +import { authProfiles } from "../../db/schema" +import { createScannerInboxService, SCANNER_DAV_PATH, SCANNER_UPLOAD_LIMIT, type ScannerInboxService } from "../modules/scanner-inbox.service" + +export default async function documentInboxRoutes(server: FastifyInstance, options: { service?: ScannerInboxService }) { + const service = options.service || createScannerInboxService(server.db) + server.addHook("preHandler", async (req, reply) => { + if (!req.user?.tenant_id) return reply.code(400).send({ error: "Bitte einen Mandanten auswählen." }) + const [profile] = await server.db.select({ portal: authProfiles.customer_for_portal }).from(authProfiles) + .where(and(eq(authProfiles.tenant_id, req.user.tenant_id), eq(authProfiles.user_id, req.user.user_id))).limit(1) + if (profile?.portal) return reply.code(403).send({ error: "Die Dokumentenablage ist nur intern verfügbar." }) + if (req.routeOptions.url?.includes("/scanner") && !req.user.is_admin) { + return reply.code(403).send({ error: "Scanner-Zugänge können nur Administratoren verwalten." }) + } + reply.header("Cache-Control", "no-store") + }) + server.get("/document-inbox", async (req, reply) => { + const query = req.query as { limit?: string; offset?: string; search?: string } + const limit = Number(query.limit ?? 50) + const offset = Number(query.offset ?? 0) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(offset) || offset < 0 || (query.search?.length || 0) > 200) { + return reply.code(400).send({ error: "Ungültige Such- oder Seiteneinstellungen." }) + } + return service.list(req.user.tenant_id!, limit, offset, query.search?.trim() || "") + }) + server.get("/document-inbox/scanner", async req => ({ + connection: await service.connection(req.user.tenant_id!), path: SCANNER_DAV_PATH, maxFileSize: SCANNER_UPLOAD_LIMIT, + })) + server.post("/document-inbox/scanner/credentials", async req => ({ + connection: await service.credentials(req.user.tenant_id!), path: SCANNER_DAV_PATH, + })) + server.delete("/document-inbox/scanner", async (_req, reply) => { + await service.disable(_req.user.tenant_id!) + return reply.code(204).send() + }) +} diff --git a/backend/src/routes/scannerInboxGateway.ts b/backend/src/routes/scannerInboxGateway.ts new file mode 100644 index 00000000..00bd4da0 --- /dev/null +++ b/backend/src/routes/scannerInboxGateway.ts @@ -0,0 +1,74 @@ +import type { FastifyInstance, FastifyRequest } from "fastify" +import { createScannerInboxService, SCANNER_UPLOAD_LIMIT, scannerError, type ScannerIdentity, type ScannerInboxService } from "../modules/scanner-inbox.service" + +const escapeXml = (text: string) => text.replace(/[<>&"']/g, char => ({ "<": "<", ">": ">", "&": "&", '"': """, "'": "'" }[char]!)) +const methods = "OPTIONS, PROPFIND, HEAD, PUT" + +export default async function scannerInboxGateway(server: FastifyInstance, options: { service?: ScannerInboxService }) { + const service = options.service || createScannerInboxService(server.db) + const identities = new WeakMap() + server.addHttpMethod("PROPFIND", { hasBody: true }) + server.addHttpMethod("MKCOL", { hasBody: true }) + // Keep raw scanner bytes, even when firmware supplies application/octet-stream or no MIME type. + server.removeAllContentTypeParsers() + server.addContentTypeParser("*", { parseAs: "buffer", bodyLimit: SCANNER_UPLOAD_LIMIT }, (_req, body, done) => done(null, body)) + + server.addHook("onRequest", async (req, reply) => { + reply.header("Cache-Control", "no-store") + const encoded = /^Basic ([A-Za-z0-9+/]+={0,2})$/i.exec(req.headers.authorization || "")?.[1] + const value = encoded && encoded.length < 1024 ? Buffer.from(encoded, "base64").toString("utf8") : "" + const separator = value.indexOf(":") + const identity = separator > 0 ? await service.authenticate(value.slice(0, separator), value.slice(separator + 1)) : null + if (!identity) return reply.header("WWW-Authenticate", 'Basic realm="FEDEO Scanner", charset="UTF-8"') + .code(401).send({ error: "Scanner-Zugangsdaten erforderlich." }) + identities.set(req, identity) + }) + + const handle = async (req: FastifyRequest, reply: any) => { + const identity = identities.get(req)! + const filename = (req.params as { "*"?: string })["*"] || "" + if (filename && (filename.length > 200 || filename.startsWith(".") || /[\\/\u0000-\u001f\u007f]/.test(filename))) { + throw scannerError(400, "Unterordner und ungültige Dateinamen werden nicht unterstützt.") + } + if (req.method === "OPTIONS") return reply.header("DAV", "1").header("Allow", methods).code(200).send() + if (req.method === "MKCOL") return reply.header("Allow", methods).code(filename ? 403 : 405).send() + if (req.method === "PUT") { + if (!filename) throw scannerError(400, "Ein Dateiname ist erforderlich.") + if (req.headers["content-range"]) throw scannerError(400, "Bitte die vollständige Datei übertragen.") + if (req.headers["if-match"] || req.headers["if-unmodified-since"] || req.headers.if) { + throw scannerError(412, "Änderungen an vorhandenen Dokumenten werden nicht unterstützt.") + } + const body = Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0) + const result = await service.accept(identity, filename, body, req.headers["if-none-match"] === "*") + return reply.header("ETag", `"${result.checksum}"`).code(result.created ? 201 : 204).send() + } + const entry = filename ? await service.find(identity.tenantId, filename) : null + if (filename && !entry) return reply.code(404).send() + if (req.method === "HEAD") { + if (entry) reply.header("Content-Length", entry.size).header("Content-Type", entry.mimeType).header("ETag", `"${entry.checksum}"`) + return reply.code(200).send() + } + if (req.method === "PROPFIND") { + // This is an upload inbox, not a browsable archive. Depth 1 deliberately exposes no other files. + if (req.headers.depth && !["0", "1"].includes(String(req.headers.depth))) { + return reply.type("application/xml").code(403).send('') + } + const prefixHeader = req.headers["x-forwarded-prefix"] + const prefix = typeof prefixHeader === "string" && /^(\/[A-Za-z0-9._~-]+)*\/?$/.test(prefixHeader) ? prefixHeader.replace(/\/$/, "") : "" + const href = prefix + req.url.split("?")[0] + (!filename && !req.url.split("?")[0].endsWith("/") ? "/" : "") + const props = entry + ? `${entry.size}${entry.mimeType}"${entry.checksum}"${entry.receivedAt.toUTCString()}` + : '' + return reply.header("DAV", "1").type("application/xml; charset=utf-8").code(207).send( + `${escapeXml(href)}${escapeXml(filename || "FEDEO Dokumentenablage")}${props}HTTP/1.1 200 OK`) + } + return reply.header("Allow", methods).code(405).send() + } + + // Both slash forms work without redirects (some scanners discard authentication on redirects). + for (const url of ["/", "/*"]) { + server.route<{}, { cors: false }>({ method: ["OPTIONS", "PROPFIND", "HEAD", "PUT", "MKCOL", "GET", "DELETE", "POST", "PATCH"], + // Scanner OPTIONS requests are WebDAV discovery, not browser CORS preflights. + url, config: { cors: false }, bodyLimit: SCANNER_UPLOAD_LIMIT, handler: handle }) + } +} diff --git a/backend/tests/scannerInbox.test.ts b/backend/tests/scannerInbox.test.ts new file mode 100644 index 00000000..e745ed69 --- /dev/null +++ b/backend/tests/scannerInbox.test.ts @@ -0,0 +1,248 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { randomUUID } from "node:crypto" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { readFile, mkdtemp, mkdir, writeFile, rm } from "node:fs/promises" +import path from "node:path" +import os from "node:os" +import Fastify from "fastify" +import { Pool } from "pg" +import { drizzle } from "drizzle-orm/node-postgres" +import { migrate } from "drizzle-orm/node-postgres/migrator" +import { getTableColumns } from "drizzle-orm" +import { PDFDocument } from "pdf-lib" +import { files } from "../db/schema" +import gateway from "../src/routes/scannerInboxGateway" +import inboxRoutes from "../src/routes/documentInbox" +import corsPlugin from "../src/plugins/cors" +import { createScannerInboxService, SCANNER_UPLOAD_LIMIT, validateScannerFile, type ScannerInboxService } from "../src/modules/scanner-inbox.service" + +const execute = promisify(execFile) +const basic = (username = "scanner", password = "secret") => ({ authorization: `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}` }) +const samplePdf = Buffer.from("%PDF-1.7\nscanner fixture\n%%EOF") + +test("scanner formats are verified independently of the supplied content type", () => { + assert.equal(validateScannerFile("scan.PDF", samplePdf), "application/pdf") + assert.equal(validateScannerFile("scan.jpg", Buffer.from([255, 216, 255, 0])), "image/jpeg") + assert.equal(validateScannerFile("scan.png", Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])), "image/png") + assert.equal(validateScannerFile("scan.tiff", Buffer.from("49492a00", "hex")), "image/tiff") + for (const [filename, body, status] of [ + ["scan.pdf", Buffer.alloc(0), 400], ["scan.pdf", Buffer.from("error"), 415], + ["scan.pdf", Buffer.from("%PDF-truncated"), 415], ["scan.zip", samplePdf, 415], + ["../scan.pdf", samplePdf, 400], ["a/b.pdf", samplePdf, 400], ["a\\b.pdf", samplePdf, 400], + [".hidden.pdf", samplePdf, 400], ["a\u0000.pdf", samplePdf, 400], + ] as const) assert.throws(() => validateScannerFile(filename, body), { statusCode: status }) + assert.throws(() => validateScannerFile("large.pdf", Buffer.alloc(SCANNER_UPLOAD_LIMIT + 1)), { statusCode: 413 }) +}) + +const protocolFixture = async () => { + const calls: any[] = [] + const service = { + authenticate: async (username: string, password: string) => username === "scanner" && password === "secret" ? { tenantId: 7 } : null, + accept: async (...args: any[]) => { calls.push(args); validateScannerFile(args[1], args[2]); return { created: true, checksum: "abc" } }, + find: async () => null, + } as unknown as ScannerInboxService + const app = Fastify() + await app.register(gateway, { prefix: "/scanner-inbox", service }) + // Match production: the global CORS plugin is registered after the scanner gateway. + await app.register(corsPlugin) + return { app, calls } +} + +test("WebDAV authentication happens before body parsing, including discovery requests", async t => { + const { app } = await protocolFixture() + t.after(() => app.close()) + for (const method of ["OPTIONS", "PROPFIND", "PUT", "HEAD"] as const) { + const res = await app.inject({ method, url: "/scanner-inbox/scan.pdf" }) + assert.equal(res.statusCode, 401) + assert.match(res.headers["www-authenticate"] as string, /^Basic /) + } + const res = await app.inject({ method: "PUT", url: "/scanner-inbox/scan.pdf", headers: { ...basic("scanner", "wrong"), "content-type": "application/json" }, payload: "not json" }) + assert.equal(res.statusCode, 401) +}) + +test("Epson discovery works with both slash forms, XML bodies and a backend proxy prefix", async t => { + const { app } = await protocolFixture() + t.after(() => app.close()) + for (const url of ["/scanner-inbox", "/scanner-inbox/"]) { + const options = await app.inject({ method: "OPTIONS", url, headers: basic() }) + assert.equal(options.statusCode, 200) + assert.match(options.headers.allow as string, /PUT/) + const head = await app.inject({ method: "HEAD", url, headers: basic() }) + assert.equal(head.statusCode, 200) + const res = await app.inject({ method: "PROPFIND", url, headers: { ...basic(), depth: "1", "content-type": "text/xml", "x-forwarded-prefix": "/backend" }, payload: '' }) + assert.equal(res.statusCode, 207, res.body) + assert.match(res.body, /\/backend\/scanner-inbox\/<\/d:href>/) + assert.match(res.body, //) + assert.equal((res.body.match(//g) || []).length, 1) + } + assert.equal((await app.inject({ method: "PROPFIND", url: "/scanner-inbox/", headers: { ...basic(), depth: "infinity" } })).statusCode, 403) +}) + +test("raw PDFs reach storage unchanged and scanner credentials cannot download or delete", async t => { + const { app, calls } = await protocolFixture() + t.after(() => app.close()) + for (const contentType of ["application/pdf", "application/octet-stream", undefined]) { + const res = await app.inject({ method: "PUT", url: "/scanner-inbox/Rechnung%20%26%20Beleg.pdf", headers: { ...basic(), ...(contentType ? { "content-type": contentType } : {}) }, payload: samplePdf }) + assert.equal(res.statusCode, 201, res.body) + assert.equal(calls.at(-1)[0].tenantId, 7) + assert.equal(calls.at(-1)[1], "Rechnung & Beleg.pdf") + assert.deepEqual(calls.at(-1)[2], samplePdf) + } + for (const method of ["GET", "DELETE", "POST", "PATCH"] as const) { + assert.equal((await app.inject({ method, url: "/scanner-inbox/", headers: basic() })).statusCode, 405) + } + for (const name of ["sub/scan.pdf", "sub%2Fscan.pdf", "..%5Cscan.pdf"]) { + assert.equal((await app.inject({ method: "PUT", url: "/scanner-inbox/" + name, headers: basic(), payload: samplePdf })).statusCode, 400) + } + assert.equal((await app.inject({ method: "PUT", url: "/scanner-inbox/scan.pdf", headers: { ...basic(), "content-range": "bytes 0-10/20" }, payload: samplePdf })).statusCode, 400) + assert.equal((await app.inject({ method: "PUT", url: "/scanner-inbox/scan.pdf", headers: { ...basic(), "if-match": '"old"' }, payload: samplePdf })).statusCode, 412) + assert.equal((await app.inject({ method: "PUT", url: "/scanner-inbox/scan.pdf", headers: { ...basic(), "content-type": "application/pdf", "content-length": String(SCANNER_UPLOAD_LIMIT + 1) }, payload: samplePdf })).statusCode, 413) +}) + +// Uses only a disposable database and fake object storage, never DATABASE_URL or production secrets. +test("scanner inbox integration: migration, persistence, isolation, retries and management", { + skip: !process.env.FEDEO_TEST_POSTGRES_IMAGE, +}, async t => { + const container = "fedeo-scanner-inbox-test-" + randomUUID() + const dbPassword = randomUUID() + await execute("docker", ["run", "--rm", "-d", "--name", container, "-e", `POSTGRES_PASSWORD=${dbPassword}`, "-p", "127.0.0.1::5432", process.env.FEDEO_TEST_POSTGRES_IMAGE!]) + let pool: Pool | undefined + const app = Fastify() + const tempMigrations = await mkdtemp(path.join(os.tmpdir(), "fedeo-scanner-inbox-")) + t.after(async () => { + await app.close() + await pool?.end() + await execute("docker", ["rm", "-f", container]) + await rm(tempMigrations, { recursive: true, force: true }) + }) + const { stdout } = await execute("docker", ["port", container, "5432/tcp"]) + pool = new Pool({ host: "127.0.0.1", port: Number(stdout.trim().split(":").at(-1)), user: "postgres", password: dbPassword, database: "postgres", connectionTimeoutMillis: 1000 }) + for (let attempt = 0; ; attempt++) { + try { await pool.query("SELECT 1"); break } catch (error) { + if (attempt >= 80) throw error + await new Promise(resolve => setTimeout(resolve, 100)) + } + } + const columns = Object.values(getTableColumns(files)).map(column => { + const suffix = column.name === "id" ? " PRIMARY KEY" : column.name === "created_at" ? " DEFAULT now()" : column.name === "archived" ? " DEFAULT false" : "" + return `"${column.name}" ${column.getSQLType()}${suffix}` + }).join(", ") + await pool.query(`CREATE TABLE tenants (id bigint PRIMARY KEY, locked text); + INSERT INTO tenants VALUES (1, null), (2, null); + CREATE TABLE files (${columns}, CHECK (name <> 'reject.pdf')); + CREATE TABLE auth_profiles (tenant_id bigint, user_id uuid, customer_for_portal bigint);`) + const migrationRoot = path.resolve(__dirname, "../db/migrations") + const journal = JSON.parse(await readFile(path.join(migrationRoot, "meta/_journal.json"), "utf8")) + const entry = journal.entries.find((item: any) => item.tag === "0073_scanner_document_inbox") + assert.ok(entry) + assert.ok(journal.entries.filter((item: any) => item.idx < entry.idx).every((item: any) => item.when < entry.when)) + await mkdir(path.join(tempMigrations, "meta")) + await writeFile(path.join(tempMigrations, "meta/_journal.json"), JSON.stringify({ ...journal, entries: [entry] })) + await writeFile(path.join(tempMigrations, entry.tag + ".sql"), await readFile(path.join(migrationRoot, entry.tag + ".sql"))) + const db = drizzle(pool) as any + await migrate(db, { migrationsFolder: tempMigrations }) + await migrate(db, { migrationsFolder: tempMigrations }) + const stored = new Map() + let storageFailure = false + const service = createScannerInboxService(db, { + async put(key, body) { stored.set(key, Buffer.from(body)); if (storageFailure) throw new Error("Simulated object store failure") }, + async remove(key) { stored.delete(key) }, + }) + const first = await service.credentials(1) + const second = await service.credentials(2) + assert.equal(first.password.length, 20, "Epson destination password limit") + assert.ok(first.username.length <= 30) + assert.equal((await service.connection(1) as any).passwordHash, undefined) + assert.equal((await pool.query("SELECT password_hash FROM scanner_inbox_connections WHERE tenant_id=1")).rows[0].password_hash.length, 64) + const userId = randomUUID() + app.decorate("db", db) + await app.register(gateway, { prefix: "/scanner-inbox", service }) + await app.register(corsPlugin) + await app.register(async api => { + api.addHook("onRequest", async req => { + req.user = { tenant_id: Number(req.headers["x-test-tenant"] || 1) || null, user_id: userId, email: "test@example.invalid", is_admin: req.headers["x-test-admin"] !== "false" } + }) + await api.register(inboxRoutes, { service }) + }, { prefix: "/api" }) + const doc = await PDFDocument.create() + doc.addPage() + doc.addPage() + const pdf = Buffer.from(await doc.save()) + const send = (name: string, payload = pdf, headers: Record = basic(first.username, first.password)) => app.inject({ method: "PUT", url: `/scanner-inbox/${encodeURIComponent(name)}`, headers: { "content-type": "application/pdf", ...headers }, payload }) + + await t.test("stores one unchanged two-page PDF as one document without extraction", async () => { + const res = await send("scan.pdf") + assert.equal(res.statusCode, 201, res.body) + const row = (await pool.query("SELECT * FROM files")).rows[0] + assert.equal(row.tenant, "1") + assert.equal(row.mimeType, "application/pdf") + assert.equal(row.extracted_text, null) + assert.deepEqual(stored.get(row.path), pdf) + assert.equal((await PDFDocument.load(stored.get(row.path)!)).getPageCount(), 2) + const listing = await app.inject({ method: "GET", url: "/api/document-inbox" }) + assert.equal(listing.json().total, 1) + assert.equal(listing.json().documents[0].name, "scan.pdf") + assert.equal((await app.inject({ method: "HEAD", url: "/scanner-inbox/scan.pdf", headers: basic(first.username, first.password) })).statusCode, 200) + }) + await t.test("serialises simultaneous retries and preserves existing content on filename conflicts", async () => { + const retry = await Promise.all([send("scan.pdf"), send("scan.pdf")]) + assert.deepEqual(retry.map(res => res.statusCode), [204, 204]) + const concurrent = await Promise.all([send("parallel.pdf"), send("parallel.pdf")]) + assert.deepEqual(concurrent.map(res => res.statusCode).sort(), [201, 204]) + assert.equal((await send("scan.pdf", samplePdf)).statusCode, 409) + assert.equal((await send("scan.pdf", pdf, { ...basic(first.username, first.password), "if-none-match": "*" })).statusCode, 412) + assert.equal(stored.size, 2) + }) + await t.test("credentials bind writes, metadata and lists to a tenant", async () => { + const otherHeaders = basic(second.username, second.password) + assert.equal((await app.inject({ method: "HEAD", url: "/scanner-inbox/scan.pdf", headers: otherHeaders })).statusCode, 404) + assert.equal((await send("scan.pdf", pdf, otherHeaders)).statusCode, 201) + const otherList = await app.inject({ method: "GET", url: "/api/document-inbox", headers: { "x-test-tenant": "2" } }) + assert.equal(otherList.json().total, 1) + const own = await app.inject({ method: "GET", url: "/api/document-inbox?limit=1&offset=1" }) + assert.equal(own.json().total, 2) + assert.equal(own.json().documents.length, 1) + const filtered = await app.inject({ method: "GET", url: "/api/document-inbox?search=parallel" }) + assert.equal(filtered.json().total, 1) + assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox?limit=-1" })).statusCode, 400) + }) + await t.test("storage and database failures leave no incomplete file or inbox entry", async () => { + const before = stored.size + storageFailure = true + assert.equal((await send("failed.pdf")).statusCode, 500) + storageFailure = false + assert.equal(stored.size, before) + assert.equal((await send("reject.pdf")).statusCode, 500) + assert.equal(stored.size, before) + assert.equal((await pool.query("SELECT count(*) FROM files WHERE name IN ('failed.pdf', 'reject.pdf')")).rows[0].count, "0") + assert.equal((await send("failed.pdf")).statusCode, 201) + }) + await t.test("maintenance, portal access and administrator-only settings are enforced", async () => { + await pool.query("UPDATE tenants SET locked='maintenance_tenant' WHERE id=1") + assert.equal((await send("maintenance.pdf")).statusCode, 423) + await pool.query("UPDATE tenants SET locked=null WHERE id=1") + assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox", headers: { "x-test-tenant": "0" } })).statusCode, 400) + for (const [method, url] of [["GET", "/api/document-inbox/scanner"], ["POST", "/api/document-inbox/scanner/credentials"], ["DELETE", "/api/document-inbox/scanner"]] as const) { + assert.equal((await app.inject({ method, url, headers: { "x-test-admin": "false" } })).statusCode, 403) + } + assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox", headers: { "x-test-admin": "false" } })).statusCode, 200) + await pool.query("INSERT INTO auth_profiles VALUES (1, $1, 123)", [userId]) + assert.equal((await app.inject({ method: "GET", url: "/api/document-inbox" })).statusCode, 403) + await pool.query("DELETE FROM auth_profiles") + }) + await t.test("revocation and rotation reject old credentials without removing documents", async () => { + const oldIdentity = await service.authenticate(first.username, first.password) + assert.equal((await app.inject({ method: "DELETE", url: "/api/document-inbox/scanner" })).statusCode, 204) + assert.equal((await send("disabled.pdf")).statusCode, 401) + const rotated = await app.inject({ method: "POST", url: "/api/document-inbox/scanner/credentials" }) + assert.equal(rotated.statusCode, 200) + assert.equal(rotated.headers["cache-control"], "no-store") + assert.equal(rotated.json().connection.username, first.username) + assert.equal((await send("old.pdf")).statusCode, 401) + await assert.rejects(service.accept(oldIdentity!, "late.pdf", pdf), { statusCode: 401 }) + assert.equal((await send("new.pdf", pdf, basic(first.username, rotated.json().connection.password))).statusCode, 201) + assert.ok((await service.list(1, 50, 0, "")).total >= 4) + }) +}) diff --git a/docs/scanner-dokumentenablage.md b/docs/scanner-dokumentenablage.md new file mode 100644 index 00000000..09399cf0 --- /dev/null +++ b/docs/scanner-dokumentenablage.md @@ -0,0 +1,41 @@ +# Epson-Scanner und Dokumentenablage + +Unter **Organisation → Dokumentenablage** erscheinen eingegangene Scanner-Dateien des aktiven Mandanten, neueste zuerst. Die Seite aktualisiert sich alle zehn Sekunden; Suche, Öffnen und Herunterladen sind verfügbar. Jede übertragene Datei wird unverändert als ein Dokument gespeichert. Ein mehrseitiges PDF bleibt ein Dokument. Es gibt keine Seitentrennung, OCR, Klassifizierung oder Rechnungsverarbeitung. + +## Einrichtung + +1. Als Administrator den gewünschten Mandanten auswählen und auf der Seite **Scanner einrichten → Scanner-Zugang erstellen** wählen. +2. Zieladresse, Benutzername und das einmalig angezeigte Passwort übernehmen. Der Zugang gehört ausschließlich zu diesem Mandanten und kann nur Scanner-Dateien einliefern. Er verwendet kein persönliches FEDEO-Passwort. Ein neues Passwort ersetzt das alte; **Zugang deaktivieren** sperrt weitere Übertragungen. +3. Am Epson ES-580W oder ES-C380W **Netzwerkordner/FTP → WebDAV (HTTPS)** wählen. In Web Config kann der Zieltyp **SharePoint (WebDAV)** heißen; dort bei separater Auswahl von HTTPS im Feld **Speichern unter** den Adressanteil ohne `https://` eintragen. Zieladresse und Zugangsdaten eintragen, Port 443 bzw. den Port der eigenen HTTPS-Adresse verwenden. Das Serverzertifikat muss vom Scanner akzeptiert werden. +4. PDF, Duplex und beispielsweise 300 dpi einstellen. Das Ziel als **FEDEO** speichern. Dateinamen mit Datum/Uhrzeit und laufender Nummer verwenden. Für getrennte Dokumente jeweils eine eigene Datei übertragen; Vorder- und Rückseite können in einer PDF bleiben. + +Die vollständige Adresse wird aus der konfigurierten Backend-Adresse gebildet, z. B. `https://fedeo.example/backend/scanner-inbox/`. Ein vorhandener `/backend`-Präfix muss erhalten bleiben. Es ist kein lokaler Rechner oder Geräte-Agent erforderlich. + +Grundlage sind die Epson-Handbücher für [ES-580W](https://download.epson.de/pub/download/6371/epson637123eu.pdf) und [ES-C380W](https://download4.epson.biz/sec_pubs/es-c380w/useg/en/GUID-55C5993A-63B2-45C9-B3C9-9C6D6801CDF0.htm). Der tatsächliche Verbindungsablauf muss bei der Inbetriebnahme am Gerät geprüft werden; es wurde kein physischer Epson-Scanner getestet. + +## Schnittstelle und Betrieb + +- Datenbankmigration `0073_scanner_document_inbox` wird über die reguläre Drizzle-Migration eingespielt. Keine neuen Umgebungsvariablen oder zusätzlichen Dienste sind erforderlich; der vorhandene S3-kompatible Dateispeicher wird genutzt. +- Der Scanner-Endpunkt liegt außerhalb der Browser-JWT-Anmeldung unter `/scanner-inbox/` und verwendet HTTP Basic mit eigenen zufälligen Zugangsdaten. Ausschließlich über HTTPS veröffentlichen. Passwörter werden nur als SHA-256-Hash gespeichert. +- Unterstützt werden `OPTIONS`, `PROPFIND` (Depth 0/1), `HEAD` und `PUT /scanner-inbox/`. Der Empfangsordner existiert bereits; Unterordner, Lesen, Löschen, Umbenennen und Überschreiben über den Scanner-Zugang sind nicht vorgesehen. Dies ist ein begrenzter WebDAV-Upload-Eingang, kein allgemeines Netzlaufwerk. Ein PROPFIND des Ordners gibt keine Dokumentenliste preis. +- `PUT` enthält genau eine vollständige Datei als Rohdaten, ohne Multipart-Formular. PDF, JPEG, PNG und TIFF werden anhand von Dateiendung und Signatur geprüft, maximal 100 MiB. Keine Prüfung der gesamten Dokumentstruktur auf Beschädigungen. +- Erfolgreiches Speichern: `201`. Identische Wiederholung unter demselben Dateinamen: `204`, ohne zweiten Ablageeintrag. Anderer Inhalt unter demselben Namen: `409`; mit `If-None-Match: *` bei vorhandenem Namen: `412`. Gleicher Inhalt unter einem neuen Namen wird als weiteres Dokument akzeptiert. +- Fehlende/ungültige/deaktivierte Zugangsdaten: `401`; leerer Upload/ungültiger Pfad: `400`; zu groß: `413`; falsches Format: `415`; Mandantenwartung: `423`. Bei Speicherfehlern gibt es keine Erfolgsbestätigung und keinen unvollständigen Ablageeintrag. Der Scanner kann erneut senden. +- Der Reverse Proxy muss die genannten HTTP-Methoden, den Authorization-Header und Uploads bis 100 MiB zulassen. Bei einem abgeschnittenen Pfadpräfix sollte er `X-Forwarded-Prefix` setzen (Traefik StripPrefix erledigt dies). HTTPS endet üblicherweise am bestehenden Proxy. +- Die Browser-API unter `/api/document-inbox` erfordert die normale FEDEO-Anmeldung und einen aktiven Mandanten. Portalbenutzer sind ausgeschlossen. Scanner-Zugänge verwalten nur Administratoren. Die Originale sind außerdem über die bestehende Dateiverwaltung zugänglich. + +## Lokaler Funktionstest + +Mit Testzugangsdaten und einer Test-PDF gegen eine eigene Instanz: + +```bash +curl --user "$SCANNER_USER:$SCANNER_PASSWORD" -X PROPFIND -H 'Depth: 0' "$SCANNER_URL" +curl --user "$SCANNER_USER:$SCANNER_PASSWORD" -H 'Content-Type: application/pdf' --upload-file test.pdf "${SCANNER_URL}test.pdf" +``` + +Die Antwort ist zunächst `201`; eine identische Wiederholung ergibt `204`. Anschließend steht genau ein Dokument in der Ablage. Die automatisierten Integrationsprüfungen verwenden eine isolierte PostgreSQL-Datenbank und einen ersetzten Objektspeicher, niemals produktive Zugangsdaten: + +```bash +cd backend +FEDEO_TEST_POSTGRES_IMAGE=postgres:16-alpine node --import tsx --test tests/scannerInbox.test.ts +``` diff --git a/frontend/components/MainNav.vue b/frontend/components/MainNav.vue index cab9fdc9..743c26b7 100644 --- a/frontend/components/MainNav.vue +++ b/frontend/components/MainNav.vue @@ -52,6 +52,11 @@ const links = computed(() => { to: "/wiki", icon: "i-heroicons-book-open" } : null, + featureEnabled("files") ? { + label: "Dokumentenablage", + to: "/organisation/dokumentenablage", + icon: "i-heroicons-inbox-arrow-down" + } : null, featureEnabled("files") ? { label: "Dateien", to: "/files", diff --git a/frontend/pages/organisation/dokumentenablage.vue b/frontend/pages/organisation/dokumentenablage.vue new file mode 100644 index 00000000..fe57cc41 --- /dev/null +++ b/frontend/pages/organisation/dokumentenablage.vue @@ -0,0 +1,247 @@ + + + diff --git a/frontend/tests/document-inbox.test.cjs b/frontend/tests/document-inbox.test.cjs new file mode 100644 index 00000000..da3cff25 --- /dev/null +++ b/frontend/tests/document-inbox.test.cjs @@ -0,0 +1,92 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const vm = require('node:vm') +const vue = require('vue') +const ts = require('../../backend/node_modules/typescript') + +const deferred = () => { + let resolve + const promise = new Promise(done => { resolve = done }) + return { promise, resolve } +} +const settle = () => new Promise(setImmediate) + +function setup(api, apiBase = 'https://fedeo.example/backend') { + const auth = vue.reactive({ activeTenant: '1', user: { is_admin: true } }) + const timers = new Map() + let timerId = 0 + let mount + let unmount + const context = vm.createContext({ + require, exports: {}, URL, + ref: vue.ref, computed: vue.computed, watch: vue.watch, + useAuthStore: () => auth, + useRuntimeConfig: () => ({ public: { apiBase } }), + useFiles: () => ({}), useToast: () => ({ add() {} }), + useNuxtApp: () => ({ $api: api }), + window: { location: { origin: 'https://fedeo.example' } }, + onMounted: handler => { mount = handler }, + onBeforeUnmount: handler => { unmount = handler }, + setTimeout: handler => { timers.set(++timerId, handler); return timerId }, + clearTimeout: id => timers.delete(id), + }) + const source = fs.readFileSync(`${__dirname}/../pages/organisation/dokumentenablage.vue`, 'utf8').match(/