From 6c613857fab0cac22eb9a13b1098f4ef2eab2e65 Mon Sep 17 00:00:00 2001 From: florianfederspiel Date: Tue, 21 Jul 2026 19:59:44 +0200 Subject: [PATCH] =?UTF-8?q?KI-AGENT:=20ZUGFeRD-=20und=20E-Rechnungserkennu?= =?UTF-8?q?ng=20erg=C3=A4nzen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/cron/prepareIncomingInvoices.ts | 29 +++- .../einvoice/detectElectronicInvoice.ts | 141 ++++++++++++++++++ backend/src/utils/fileBuffer.ts | 26 ++++ backend/src/utils/gpt.ts | 28 +--- 4 files changed, 200 insertions(+), 24 deletions(-) create mode 100644 backend/src/modules/einvoice/detectElectronicInvoice.ts create mode 100644 backend/src/utils/fileBuffer.ts diff --git a/backend/src/modules/cron/prepareIncomingInvoices.ts b/backend/src/modules/cron/prepareIncomingInvoices.ts index 4582d2a..98cbf8d 100644 --- a/backend/src/modules/cron/prepareIncomingInvoices.ts +++ b/backend/src/modules/cron/prepareIncomingInvoices.ts @@ -1,6 +1,8 @@ import { FastifyInstance } from "fastify" import dayjs from "dayjs" import { getInvoiceDataFromGPT } from "../../utils/gpt" +import { loadFileBuffer } from "../../utils/fileBuffer" +import { detectElectronicInvoice } from "../einvoice/detectElectronicInvoice" // Drizzle schema import { @@ -90,7 +92,32 @@ export function prepareIncomingInvoices(server: FastifyInstance) { for (const file of filesRes) { console.log(`Processing file ${file.id} for tenant ${tenantId}`) - const data = await getInvoiceDataFromGPT(server,file, tenantId) + let fileData: Buffer + + try { + fileData = await loadFileBuffer(file.path!) + } catch (error) { + server.log.error(error, `Datei ${file.id} konnte nicht aus S3 geladen werden.`) + continue + } + + const electronicInvoice = await detectElectronicInvoice(fileData, file) + + if (electronicInvoice) { + server.log.info({ + fileId: file.id, + container: electronicInvoice.container, + syntax: electronicInvoice.syntax, + attachmentName: electronicInvoice.attachmentName, + }, "Strukturierte E-Rechnung erkannt.") + + if (electronicInvoice.container === "xml") { + server.log.warn({ fileId: file.id }, "Die Auswertung eigenständiger E-Rechnungs-XML folgt in Etappe 2.") + continue + } + } + + const data = await getInvoiceDataFromGPT(server, file, tenantId, fileData) if (!data) { server.log.warn(`GPT returned no data for file ${file.id}`) diff --git a/backend/src/modules/einvoice/detectElectronicInvoice.ts b/backend/src/modules/einvoice/detectElectronicInvoice.ts new file mode 100644 index 0000000..5b174fe --- /dev/null +++ b/backend/src/modules/einvoice/detectElectronicInvoice.ts @@ -0,0 +1,141 @@ +import { + decodePDFRawStream, + PDFDict, + PDFDocument, + PDFHexString, + PDFName, + PDFRawStream, + PDFString, +} from "pdf-lib" + +export type ElectronicInvoiceSyntax = "cii" | "ubl-invoice" | "ubl-credit-note" + +export type ElectronicInvoiceDetection = { + container: "pdf" | "xml" + syntax: ElectronicInvoiceSyntax + xml: Buffer + attachmentName: string | null +} + +type FileLike = { + name?: string | null + path?: string | null + mimeType?: string | null +} + +const decodePdfText = (value: unknown): string | null => { + if (value instanceof PDFString || value instanceof PDFHexString) { + return value.decodeText() + } + + return null +} + +const detectXmlSyntax = (xml: Buffer): ElectronicInvoiceSyntax | null => { + const sample = xml.subarray(0, Math.min(xml.length, 64 * 1024)).toString("utf8") + + if (/<(?:[A-Za-z_][\w.-]*:)?CrossIndustryInvoice(?:\s|>)/i.test(sample)) { + return "cii" + } + + if ( + /)/i.test(sample) + && /urn:oasis:names:specification:ubl:schema:xsd:Invoice-2/i.test(sample) + ) { + return "ubl-invoice" + } + + if ( + /)/i.test(sample) + && /urn:oasis:names:specification:ubl:schema:xsd:CreditNote-2/i.test(sample) + ) { + return "ubl-credit-note" + } + + return null +} + +const extractXmlFromFileSpec = (fileSpec: PDFDict): { name: string | null, xml: Buffer } | null => { + const context = fileSpec.context + const embeddedFiles = context.lookupMaybe(fileSpec.get(PDFName.of("EF")), PDFDict) + + if (!embeddedFiles) return null + + const stream = context.lookup( + embeddedFiles.get(PDFName.of("UF")) || embeddedFiles.get(PDFName.of("F")), + ) + + if (!(stream instanceof PDFRawStream)) return null + + const name = decodePdfText( + context.lookup(fileSpec.get(PDFName.of("UF")) || fileSpec.get(PDFName.of("F"))), + ) + const contents = Buffer.from(decodePDFRawStream(stream).decode()) + + return { name, xml: contents } +} + +const detectEmbeddedInvoice = async (pdf: Buffer): Promise => { + let document: PDFDocument + + try { + document = await PDFDocument.load(pdf, { + ignoreEncryption: true, + updateMetadata: false, + }) + } catch { + return null + } + + for (const [, object] of document.context.enumerateIndirectObjects()) { + if (!(object instanceof PDFDict)) continue + + const type = object.get(PDFName.of("Type")) + if (!(type instanceof PDFName) || type.asString() !== "/Filespec") continue + + const embedded = extractXmlFromFileSpec(object) + if (!embedded) continue + + const syntax = detectXmlSyntax(embedded.xml) + if (!syntax) continue + + return { + container: "pdf", + syntax, + xml: embedded.xml, + attachmentName: embedded.name, + } + } + + return null +} + +const hasPdfFileType = (file: FileLike, data: Buffer) => { + const filename = String(file.name || file.path || "").toLowerCase() + const mimeType = String(file.mimeType || "").toLowerCase().split(";")[0].trim() + + return filename.endsWith(".pdf") || mimeType === "application/pdf" || data.subarray(0, 5).toString("ascii") === "%PDF-" +} + +export const detectElectronicInvoice = async ( + data: Buffer, + file: FileLike, +): Promise => { + if (hasPdfFileType(file, data)) { + return detectEmbeddedInvoice(data) + } + + const beginsLikeXml = data.subarray(0, 1024).toString("utf8").replace(/^\uFEFF/, "").trimStart().startsWith("<") + const syntax = beginsLikeXml ? detectXmlSyntax(data) : null + + if (syntax) { + return { + container: "xml", + syntax, + xml: data, + attachmentName: file.name || file.path?.split("/").pop() || null, + } + } + + return null +} diff --git a/backend/src/utils/fileBuffer.ts b/backend/src/utils/fileBuffer.ts new file mode 100644 index 0000000..a6d9b4e --- /dev/null +++ b/backend/src/utils/fileBuffer.ts @@ -0,0 +1,26 @@ +import { GetObjectCommand } from "@aws-sdk/client-s3" +import { s3 } from "./s3" +import { secrets } from "./secrets" + +export const streamToBuffer = async (stream: any): Promise => { + const chunks: Buffer[] = [] + + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + + return Buffer.concat(chunks) +} + +export const loadFileBuffer = async (path: string): Promise => { + const response = await s3.send(new GetObjectCommand({ + Bucket: secrets.S3_BUCKET, + Key: path, + })) + + if (!response.Body) { + throw new Error(`S3-Datei '${path}' enthält keinen lesbaren Inhalt.`) + } + + return streamToBuffer(response.Body) +} diff --git a/backend/src/utils/gpt.ts b/backend/src/utils/gpt.ts index 95b9cc7..3577faa 100644 --- a/backend/src/utils/gpt.ts +++ b/backend/src/utils/gpt.ts @@ -2,12 +2,11 @@ import dayjs from "dayjs"; import OpenAI from "openai"; import { z } from "zod"; import { zodResponseFormat } from "openai/helpers/zod"; -import { GetObjectCommand } from "@aws-sdk/client-s3"; import { FastifyInstance } from "fastify"; -import { s3 } from "./s3"; -import { secrets } from "./secrets"; import { storeExtractedTextForFile } from "./documentText"; +import { loadFileBuffer } from "./fileBuffer"; +import { secrets } from "./secrets"; // Drizzle schema import { vendors, accounts, tenants } from "../../db/schema"; @@ -27,18 +26,6 @@ export const initOpenAi = async () => { }); }; -// --------------------------------------------------------- -// STREAM → BUFFER -// --------------------------------------------------------- -async function streamToBuffer(stream: any): Promise { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - stream.on("data", (chunk: Buffer) => chunks.push(chunk)); - stream.on("error", reject); - stream.on("end", () => resolve(Buffer.concat(chunks))); - }); -} - // --------------------------------------------------------- // GPT RESPONSE FORMAT (Zod Schema) // --------------------------------------------------------- @@ -93,7 +80,8 @@ const InstructionFormat = z.object({ export const getInvoiceDataFromGPT = async function ( server: FastifyInstance, file: any, - tenantId: number + tenantId: number, + suppliedFileData?: Buffer, ) { await initOpenAi(); @@ -109,13 +97,7 @@ export const getInvoiceDataFromGPT = async function ( let fileData: Buffer; try { - const command = new GetObjectCommand({ - Bucket: secrets.S3_BUCKET, - Key: file.path, - }); - - const response: any = await s3.send(command); - fileData = await streamToBuffer(response.Body); + fileData = suppliedFileData || await loadFileBuffer(file.path); } catch (err) { console.log(`❌ S3 Download failed for file ${file.id}`, err); return null;