diff --git a/package.json b/package.json index f188720..c657240 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "imapflow": "^1.1.1", "jsonwebtoken": "^9.0.2", "nodemailer": "^7.0.6", + "openai": "^6.10.0", "pdf-lib": "^1.17.1", "pg": "^8.16.3", "pngjs": "^7.0.0", diff --git a/src/modules/cron/prepareIncomingInvoices.ts b/src/modules/cron/prepareIncomingInvoices.ts new file mode 100644 index 0000000..011c1a3 --- /dev/null +++ b/src/modules/cron/prepareIncomingInvoices.ts @@ -0,0 +1,175 @@ +import { FastifyInstance } from "fastify" +import dayjs from "dayjs" +import { getInvoiceDataFromGPT } from "../../utils/gpt" + +// Drizzle schema +import { + tenants, + files, + filetags, + incominginvoices, +} from "../../../db/schema" + +import { eq, and, isNull, not } from "drizzle-orm" + +export function prepareIncomingInvoices(server: FastifyInstance) { + const processInvoices = async (tenantId:number) => { + console.log("▶ Starting Incoming Invoice Preparation") + + const tenantsRes = await server.db + .select() + .from(tenants) + .where(eq(tenants.id, tenantId)) + .orderBy(tenants.id) + + if (!tenantsRes.length) { + console.log("No tenants with autoPrepareIncomingInvoices = true") + return + } + + console.log(`Processing tenants: ${tenantsRes.map(t => t.id).join(", ")}`) + + // ------------------------------------------------------------- + // 2️⃣ Jeden Tenant einzeln verarbeiten + // ------------------------------------------------------------- + for (const tenant of tenantsRes) { + const tenantId = tenant.id + + // 2.1 Datei-Tags holen für incoming invoices + const tagRes = await server.db + .select() + .from(filetags) + .where( + and( + eq(filetags.tenant, tenantId), + eq(filetags.incomingDocumentType, "invoices") + ) + ) + .limit(1) + + const invoiceFileTag = tagRes?.[0]?.id + if (!invoiceFileTag) { + server.log.error(`❌ Missing filetag 'invoices' for tenant ${tenantId}`) + continue + } + + // 2.2 Alle Dateien laden, die als Invoice markiert sind aber NOCH keine incominginvoice haben + const filesRes = await server.db + .select() + .from(files) + .where( + and( + eq(files.tenant, tenantId), + eq(files.type, invoiceFileTag), + isNull(files.incominginvoice), + eq(files.archived, false), + not(isNull(files.path)) + ) + ) + + if (!filesRes.length) { + console.log(`No invoice files for tenant ${tenantId}`) + continue + } + + // ------------------------------------------------------------- + // 3️⃣ Jede Datei einzeln durch GPT jagen & IncomingInvoice erzeugen + // ------------------------------------------------------------- + for (const file of filesRes) { + console.log(`Processing file ${file.id} for tenant ${tenantId}`) + + const data = await getInvoiceDataFromGPT(server,file, tenantId) + + if (!data) { + server.log.warn(`GPT returned no data for file ${file.id}`) + continue + } + + // --------------------------------------------------------- + // 3.1 IncomingInvoice-Objekt vorbereiten + // --------------------------------------------------------- + let itemInfo: any = { + tenant: tenantId, + state: "Vorbereitet" + } + + if (data.invoice_number) itemInfo.reference = data.invoice_number + if (data.invoice_date) itemInfo.date = dayjs(data.invoice_date).toISOString() + if (data.issuer?.id) itemInfo.vendor = data.issuer.id + if (data.invoice_duedate) itemInfo.dueDate = dayjs(data.invoice_duedate).toISOString() + + // Payment terms mapping + const mapPayment: any = { + "Direct Debit": "Einzug", + "Transfer": "Überweisung", + "Credit Card": "Kreditkarte", + "Other": "Sonstiges", + } + if (data.terms) itemInfo.paymentType = mapPayment[data.terms] ?? data.terms + + // 3.2 Positionszeilen konvertieren + if (data.invoice_items?.length > 0) { + itemInfo.accounts = data.invoice_items.map(item => ({ + account: item.account_id, + description: item.description, + amountNet: item.total_without_tax, + amountTax: Number((item.total - item.total_without_tax).toFixed(2)), + taxType: String(item.tax_rate), + amountGross: item.total, + costCentre: null, + quantity: item.quantity, + })) + } + + // 3.3 Beschreibung generieren + let description = "" + if (data.delivery_note_number) description += `Lieferschein: ${data.delivery_note_number}\n` + if (data.reference) description += `Referenz: ${data.reference}\n` + if (data.invoice_items) { + for (const item of data.invoice_items) { + description += `${item.description} - ${item.quantity} ${item.unit} - ${item.total}\n` + } + } + itemInfo.description = description.trim() + + // --------------------------------------------------------- + // 4️⃣ IncomingInvoice erstellen + // --------------------------------------------------------- + const inserted = await server.db + .insert(incominginvoices) + .values(itemInfo) + .returning() + + const newInvoice = inserted?.[0] + + if (!newInvoice) { + server.log.error(`Failed to insert incoming invoice for file ${file.id}`) + continue + } + + // --------------------------------------------------------- + // 5️⃣ Datei mit incominginvoice-ID verbinden + // --------------------------------------------------------- + await server.db + .update(files) + .set({ incominginvoice: newInvoice.id }) + .where(eq(files.id, file.id)) + + console.log(`IncomingInvoice ${newInvoice.id} created for file ${file.id}`) + } + + } + + + return + } + + return { + run: async (tenant:number) => { + await processInvoices(tenant) + console.log("Incoming Invoice Preparation Completed.") + + } + } + +} diff --git a/src/modules/serialexecution.service.ts b/src/modules/serialexecution.service.ts index 8f78b5d..82a98ff 100644 --- a/src/modules/serialexecution.service.ts +++ b/src/modules/serialexecution.service.ts @@ -388,15 +388,19 @@ async function getCloseData(server:FastifyInstance,item: any, tenant: any, units console.log(item); - const [contact, customer, profile, project, contract] = await Promise.all([ + const [contact, customer, project, contract] = await Promise.all([ fetchById(server, schema.contacts, item.contact), fetchById(server, schema.customers, item.customer), - fetchById(server, schema.authProfiles, item.contactPerson), // oder createdBy, je nach Logik fetchById(server, schema.projects, item.project), fetchById(server, schema.contracts, item.contract), item.letterhead ? fetchById(server, schema.letterheads, item.letterhead) : null + ]); + const profile = (await server.db.select().from(schema.authProfiles).where(and(eq(schema.authProfiles.user_id, item.created_by),eq(schema.authProfiles.tenant_id,tenant.id))).limit(1))[0]; + + console.log(profile) + const pdfData = getDocumentDataBackend( { ...item, @@ -613,6 +617,8 @@ export function getDocumentDataBackend( `${itemInfo.address?.zip || customerData.zip} ${itemInfo.address?.city || customerData.city}`, ].filter(Boolean); // Leere Einträge entfernen + console.log(contactPerson) + // Info Block aufbereiten const infoBlock = [ { @@ -634,12 +640,12 @@ export function getDocumentDataBackend( }] : []), { label: "Ansprechpartner", - content: contactPerson ? (contactPerson.name || contactPerson.fullName || contactPerson.email) : "-", + content: contactPerson ? (contactPerson.name || contactPerson.full_name || contactPerson.email) : "-", }, // Kontakt Infos - ...((itemInfo.contactTel || contactPerson?.fixedTel || contactPerson?.mobileTel) ? [{ + ...((itemInfo.contactTel || contactPerson?.fixed_tel || contactPerson?.mobile_tel) ? [{ label: "Telefon", - content: itemInfo.contactTel || contactPerson?.fixedTel || contactPerson?.mobileTel, + content: itemInfo.contactTel || contactPerson?.fixed_tel || contactPerson?.mobile_tel, }] : []), ...(contactPerson?.email ? [{ label: "E-Mail", diff --git a/src/plugins/services.ts b/src/plugins/services.ts index 1c9429c..c5788b7 100644 --- a/src/plugins/services.ts +++ b/src/plugins/services.ts @@ -3,14 +3,14 @@ import fp from "fastify-plugin"; import { bankStatementService } from "../modules/cron/bankstatementsync.service"; //import {initDokuboxClient, syncDokubox} from "../modules/cron/dokuboximport.service"; import { FastifyInstance } from "fastify"; -//import {prepareIncomingInvoices} from "../modules/cron/prepareIncomingInvoices"; +import {prepareIncomingInvoices} from "../modules/cron/prepareIncomingInvoices"; declare module "fastify" { interface FastifyInstance { services: { bankStatements: ReturnType; //dokuboxSync: ReturnType; - //prepareIncomingInvoices: ReturnType; + prepareIncomingInvoices: ReturnType; }; } } @@ -19,6 +19,6 @@ export default fp(async function servicePlugin(server: FastifyInstance) { server.decorate("services", { bankStatements: bankStatementService(server), //dokuboxSync: syncDokubox(server), - //prepareIncomingInvoices: prepareIncomingInvoices(server), + prepareIncomingInvoices: prepareIncomingInvoices(server), }); }); diff --git a/src/routes/functions.ts b/src/routes/functions.ts index 67a6582..d37963b 100644 --- a/src/routes/functions.ts +++ b/src/routes/functions.ts @@ -174,6 +174,11 @@ export default async function functionRoutes(server: FastifyInstance) { await server.services.bankStatements.run(req.user.tenant_id); }) + server.post('/functions/services/prepareincominginvoices', async (req, reply) => { + + await server.services.prepareIncomingInvoices.run(req.user.tenant_id) + }) + /*server.post('/print/zpl/preview', async (req, reply) => { const { zpl, widthMm = 50, heightMm = 30, dpmm = 8, asBase64 = false } = req.body as {zpl:string,widthMm:number,heightMm:number,dpmm:number,asBase64:string} diff --git a/src/utils/gpt.ts b/src/utils/gpt.ts new file mode 100644 index 0000000..ec804ce --- /dev/null +++ b/src/utils/gpt.ts @@ -0,0 +1,204 @@ +import dayjs from "dayjs"; +import axios from "axios"; +import OpenAI from "openai"; +import { z } from "zod"; +import { zodResponseFormat } from "openai/helpers/zod"; +import { GetObjectCommand } from "@aws-sdk/client-s3"; +import { Blob } from "buffer"; +import { FastifyInstance } from "fastify"; + +import { s3 } from "./s3"; +import { secrets } from "./secrets"; + +// Drizzle schema +import { vendors, accounts } from "../../db/schema"; +import {eq} from "drizzle-orm"; + +let openai: OpenAI | null = null; + +// --------------------------------------------------------- +// INITIALIZE OPENAI +// --------------------------------------------------------- +export const initOpenAi = async () => { + openai = new OpenAI({ + apiKey: secrets.OPENAI_API_KEY, + }); +}; + +// --------------------------------------------------------- +// STREAM → BUFFER +// --------------------------------------------------------- +async function streamToBuffer(stream: any): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("error", reject); + stream.on("end", () => resolve(Buffer.concat(chunks))); + }); +} + +// --------------------------------------------------------- +// GPT RESPONSE FORMAT (Zod Schema) +// --------------------------------------------------------- +const InstructionFormat = z.object({ + invoice_number: z.string(), + invoice_date: z.string(), + invoice_duedate: z.string(), + invoice_type: z.string(), + delivery_type: z.string(), + delivery_note_number: z.string(), + reference: z.string(), + issuer: z.object({ + id: z.number().nullable().optional(), + name: z.string(), + address: z.string(), + phone: z.string(), + email: z.string(), + bank: z.string(), + bic: z.string(), + iban: z.string(), + }), + recipient: z.object({ + name: z.string(), + address: z.string(), + phone: z.string(), + email: z.string(), + }), + invoice_items: z.array( + z.object({ + description: z.string(), + unit: z.string(), + quantity: z.number(), + total: z.number(), + total_without_tax: z.number(), + tax_rate: z.number(), + ean: z.number().nullable().optional(), + article_number: z.number().nullable().optional(), + account_number: z.number().nullable().optional(), + account_id: z.number().nullable().optional(), + }) + ), + subtotal: z.number(), + tax_rate: z.number(), + tax: z.number(), + total: z.number(), + terms: z.string(), +}); + +// --------------------------------------------------------- +// MAIN FUNCTION – REPLACES SUPABASE VERSION +// --------------------------------------------------------- +export const getInvoiceDataFromGPT = async function ( + server: FastifyInstance, + file: any, + tenantId: number +) { + await initOpenAi(); + + if (!openai) { + throw new Error("OpenAI not initialized. Call initOpenAi() first."); + } + + console.log(`📄 Reading invoice file ${file.id}`); + + // --------------------------------------------------------- + // 1) DOWNLOAD PDF FROM S3 + // --------------------------------------------------------- + let fileData: Buffer; + + try { + const command = new GetObjectCommand({ + Bucket: secrets.S3_BUCKET, + Key: file.path, + }); + + const response: any = await s3.send(command); + fileData = await streamToBuffer(response.Body); + } catch (err) { + console.log(`❌ S3 Download failed for file ${file.id}`, err); + return null; + } + + // Only process PDFs + if (!file.path.toLowerCase().endsWith(".pdf")) { + server.log.warn(`Skipping non-PDF file ${file.id}`); + return null; + } + + const fileBlob = new Blob([fileData], { type: "application/pdf" }); + + // --------------------------------------------------------- + // 2) SEND FILE TO PDF → TEXT API + // --------------------------------------------------------- + const form = new FormData(); + form.append("fileInput", fileBlob, file.path.split("/").pop()); + form.append("outputFormat", "txt"); + + let extractedText: string; + + try { + const res = await axios.post( + "http://23.88.52.85:8080/api/v1/convert/pdf/text", + form, + { + headers: { + "Content-Type": "multipart/form-data", + Authorization: `Bearer ${secrets.STIRLING_API_KEY}`, + }, + } + ); + + extractedText = res.data; + } catch (err) { + console.log("❌ PDF OCR API failed", err); + return null; + } + + // --------------------------------------------------------- + // 3) LOAD VENDORS + ACCOUNTS (DRIZZLE) + // --------------------------------------------------------- + const vendorList = await server.db + .select({ id: vendors.id, name: vendors.name }) + .from(vendors) + .where(eq(vendors.tenant,tenantId)); + + const accountList = await server.db + .select({ + id: accounts.id, + label: accounts.label, + number: accounts.number, + }) + .from(accounts); + + // --------------------------------------------------------- + // 4) GPT ANALYSIS + // --------------------------------------------------------- + + + + const completion = await openai.chat.completions.parse({ + model: "gpt-4o", + store: true, + response_format: zodResponseFormat(InstructionFormat as any, "instruction"), + messages: [ + { role: "user", content: extractedText }, + { + role: "user", + content: + "You extract structured invoice data.\n\n" + + `VENDORS: ${JSON.stringify(vendorList)}\n` + + `ACCOUNTS: ${JSON.stringify(accountList)}\n\n` + + "Match issuer by name to vendor.id.\n" + + "Match invoice items to account id based on label/number.\n" + + "Convert dates to YYYY-MM-DD.\n" + + "Keep invoice items in original order.\n", + }, + ], + }); + + const parsed = completion.choices[0].message.parsed; + + console.log(`🧾 Extracted invoice data for file ${file.id}`); + + return parsed; +}; diff --git a/src/utils/secrets.ts b/src/utils/secrets.ts index b67a1e7..c21e2d7 100644 --- a/src/utils/secrets.ts +++ b/src/utils/secrets.ts @@ -33,6 +33,13 @@ export let secrets = { GOCARDLESS_BASE_URL: string GOCARDLESS_SECRET_ID: string GOCARDLESS_SECRET_KEY: string + DOKUBOX_IMAP_HOST: string + DOKUBOX_IMAP_PORT: number + DOKUBOX_IMAP_SECURE: boolean + DOKUBOX_IMAP_USER: string + DOKUBOX_IMAP_PASSWORD: string + OPENAI_API_KEY: string + STIRLING_API_KEY: string } export async function loadSecrets () {