Merge branch 'orm' into 'main'

Orm

See merge request fedeo/backend!17

package.json

@@ -42,6 +42,7 @@
     "imapflow": "^1.1.1",
     "jsonwebtoken": "^9.0.2",
     "nodemailer": "^7.0.6",
+    "openai": "^6.10.0",
     "pdf-lib": "^1.17.1",
     "pg": "^8.16.3",
     "pngjs": "^7.0.0",

src/modules/cron/prepareIncomingInvoices.ts

@@ -0,0 +1,175 @@
+import { FastifyInstance } from "fastify"
+import dayjs from "dayjs"
+import { getInvoiceDataFromGPT } from "../../utils/gpt"
+
+// Drizzle schema
+import {
+    tenants,
+    files,
+    filetags,
+    incominginvoices,
+} from "../../../db/schema"
+
+import { eq, and, isNull, not } from "drizzle-orm"
+
+export function prepareIncomingInvoices(server: FastifyInstance) {
+    const processInvoices = async (tenantId:number) => {
+        console.log("▶ Starting Incoming Invoice Preparation")
+
+        const tenantsRes = await server.db
+            .select()
+            .from(tenants)
+            .where(eq(tenants.id, tenantId))
+            .orderBy(tenants.id)
+
+        if (!tenantsRes.length) {
+            console.log("No tenants with autoPrepareIncomingInvoices = true")
+            return
+        }
+
+        console.log(`Processing tenants: ${tenantsRes.map(t => t.id).join(", ")}`)
+
+        // -------------------------------------------------------------
+        // 2️⃣ Jeden Tenant einzeln verarbeiten
+        // -------------------------------------------------------------
+        for (const tenant of tenantsRes) {
+            const tenantId = tenant.id
+
+            // 2.1 Datei-Tags holen für incoming invoices
+            const tagRes = await server.db
+                .select()
+                .from(filetags)
+                .where(
+                    and(
+                        eq(filetags.tenant, tenantId),
+                        eq(filetags.incomingDocumentType, "invoices")
+                    )
+                )
+                .limit(1)
+
+            const invoiceFileTag = tagRes?.[0]?.id
+            if (!invoiceFileTag) {
+                server.log.error(`❌ Missing filetag 'invoices' for tenant ${tenantId}`)
+                continue
+            }
+
+            // 2.2 Alle Dateien laden, die als Invoice markiert sind aber NOCH keine incominginvoice haben
+            const filesRes = await server.db
+                .select()
+                .from(files)
+                .where(
+                    and(
+                        eq(files.tenant, tenantId),
+                        eq(files.type, invoiceFileTag),
+                        isNull(files.incominginvoice),
+                        eq(files.archived, false),
+                        not(isNull(files.path))
+                    )
+                )
+
+            if (!filesRes.length) {
+                console.log(`No invoice files for tenant ${tenantId}`)
+                continue
+            }
+
+            // -------------------------------------------------------------
+            // 3️⃣ Jede Datei einzeln durch GPT jagen & IncomingInvoice erzeugen
+            // -------------------------------------------------------------
+            for (const file of filesRes) {
+                console.log(`Processing file ${file.id} for tenant ${tenantId}`)
+
+                const data = await getInvoiceDataFromGPT(server,file, tenantId)
+
+                if (!data) {
+                    server.log.warn(`GPT returned no data for file ${file.id}`)
+                    continue
+                }
+
+                // ---------------------------------------------------------
+                // 3.1 IncomingInvoice-Objekt vorbereiten
+                // ---------------------------------------------------------
+                let itemInfo: any = {
+                    tenant: tenantId,
+                    state: "Vorbereitet"
+                }
+
+                if (data.invoice_number) itemInfo.reference = data.invoice_number
+                if (data.invoice_date) itemInfo.date = dayjs(data.invoice_date).toISOString()
+                if (data.issuer?.id) itemInfo.vendor = data.issuer.id
+                if (data.invoice_duedate) itemInfo.dueDate = dayjs(data.invoice_duedate).toISOString()
+
+                // Payment terms mapping
+                const mapPayment: any = {
+                    "Direct Debit": "Einzug",
+                    "Transfer": "Überweisung",
+                    "Credit Card": "Kreditkarte",
+                    "Other": "Sonstiges",
+                }
+                if (data.terms) itemInfo.paymentType = mapPayment[data.terms] ?? data.terms
+
+                // 3.2 Positionszeilen konvertieren
+                if (data.invoice_items?.length > 0) {
+                    itemInfo.accounts = data.invoice_items.map(item => ({
+                        account: item.account_id,
+                        description: item.description,
+                        amountNet: item.total_without_tax,
+                        amountTax: Number((item.total - item.total_without_tax).toFixed(2)),
+                        taxType: String(item.tax_rate),
+                        amountGross: item.total,
+                        costCentre: null,
+                        quantity: item.quantity,
+                    }))
+                }
+
+                // 3.3 Beschreibung generieren
+                let description = ""
+                if (data.delivery_note_number) description += `Lieferschein: ${data.delivery_note_number}\n`
+                if (data.reference) description += `Referenz: ${data.reference}\n`
+                if (data.invoice_items) {
+                    for (const item of data.invoice_items) {
+                        description += `${item.description} - ${item.quantity} ${item.unit} - ${item.total}\n`
+                    }
+                }
+                itemInfo.description = description.trim()
+
+                // ---------------------------------------------------------
+                // 4️⃣ IncomingInvoice erstellen
+                // ---------------------------------------------------------
+                const inserted = await server.db
+                    .insert(incominginvoices)
+                    .values(itemInfo)
+                    .returning()
+
+                const newInvoice = inserted?.[0]
+
+                if (!newInvoice) {
+                    server.log.error(`Failed to insert incoming invoice for file ${file.id}`)
+                    continue
+                }
+
+                // ---------------------------------------------------------
+                // 5️⃣ Datei mit incominginvoice-ID verbinden
+                // ---------------------------------------------------------
+                await server.db
+                    .update(files)
+                    .set({ incominginvoice: newInvoice.id })
+                    .where(eq(files.id, file.id))
+
+                console.log(`IncomingInvoice ${newInvoice.id} created for file ${file.id}`)
+            }
+
+        }
+
+
+        return
+    }
+
+    return {
+        run: async (tenant:number) => {
+            await processInvoices(tenant)
+            console.log("Incoming Invoice Preparation Completed.")
+
+        }
+    }
+
+}

src/modules/serialexecution.service.ts

@@ -388,15 +388,19 @@ async function getCloseData(server:FastifyInstance,item: any, tenant: any, units
 
     console.log(item);
 
-    const [contact, customer, profile, project, contract] = await Promise.all([
+    const [contact, customer, project, contract] = await Promise.all([
         fetchById(server, schema.contacts, item.contact),
         fetchById(server, schema.customers, item.customer),
-        fetchById(server, schema.authProfiles, item.contactPerson), // oder createdBy, je nach Logik
         fetchById(server, schema.projects, item.project),
         fetchById(server, schema.contracts, item.contract),
         item.letterhead ? fetchById(server, schema.letterheads, item.letterhead) : null
+
     ]);
 
+    const profile = (await server.db.select().from(schema.authProfiles).where(and(eq(schema.authProfiles.user_id, item.created_by),eq(schema.authProfiles.tenant_id,tenant.id))).limit(1))[0];
+
+    console.log(profile)
+
     const pdfData = getDocumentDataBackend(
         {
             ...item,
@@ -613,6 +617,8 @@ export function getDocumentDataBackend(
         `${itemInfo.address?.zip || customerData.zip} ${itemInfo.address?.city || customerData.city}`,
     ].filter(Boolean); // Leere Einträge entfernen
 
+    console.log(contactPerson)
+
     // Info Block aufbereiten
     const infoBlock = [
         {
@@ -634,12 +640,12 @@ export function getDocumentDataBackend(
         }] : []),
         {
             label: "Ansprechpartner",
-            content: contactPerson ? (contactPerson.name || contactPerson.fullName || contactPerson.email) : "-",
+            content: contactPerson ? (contactPerson.name || contactPerson.full_name || contactPerson.email) : "-",
         },
         // Kontakt Infos
-        ...((itemInfo.contactTel || contactPerson?.fixedTel || contactPerson?.mobileTel) ? [{
+        ...((itemInfo.contactTel || contactPerson?.fixed_tel || contactPerson?.mobile_tel) ? [{
             label: "Telefon",
-            content: itemInfo.contactTel || contactPerson?.fixedTel || contactPerson?.mobileTel,
+            content: itemInfo.contactTel || contactPerson?.fixed_tel || contactPerson?.mobile_tel,
         }] : []),
         ...(contactPerson?.email ? [{
             label: "E-Mail",

src/plugins/services.ts

@@ -3,14 +3,14 @@ import fp from "fastify-plugin";
 import { bankStatementService } from "../modules/cron/bankstatementsync.service";
 //import {initDokuboxClient, syncDokubox} from "../modules/cron/dokuboximport.service";
 import { FastifyInstance } from "fastify";
-//import {prepareIncomingInvoices} from "../modules/cron/prepareIncomingInvoices";
+import {prepareIncomingInvoices} from "../modules/cron/prepareIncomingInvoices";
 
 declare module "fastify" {
     interface FastifyInstance {
         services: {
             bankStatements: ReturnType<typeof bankStatementService>;
             //dokuboxSync: ReturnType<typeof syncDokubox>;
-            //prepareIncomingInvoices: ReturnType<typeof prepareIncomingInvoices>;
+            prepareIncomingInvoices: ReturnType<typeof prepareIncomingInvoices>;
         };
     }
 }
@@ -19,6 +19,6 @@ export default fp(async function servicePlugin(server: FastifyInstance) {
     server.decorate("services", {
         bankStatements: bankStatementService(server),
         //dokuboxSync: syncDokubox(server),
-        //prepareIncomingInvoices: prepareIncomingInvoices(server),
+        prepareIncomingInvoices: prepareIncomingInvoices(server),
     });
 });

src/routes/functions.ts

@@ -174,6 +174,11 @@ export default async function functionRoutes(server: FastifyInstance) {
         await server.services.bankStatements.run(req.user.tenant_id);
     })
 
+    server.post('/functions/services/prepareincominginvoices', async (req, reply) => {
+
+        await server.services.prepareIncomingInvoices.run(req.user.tenant_id)
+    })
+
 
     /*server.post('/print/zpl/preview', async (req, reply) => {
         const { zpl, widthMm = 50, heightMm = 30, dpmm = 8, asBase64 = false } = req.body as {zpl:string,widthMm:number,heightMm:number,dpmm:number,asBase64:string}

src/utils/gpt.ts

@@ -0,0 +1,204 @@
+import dayjs from "dayjs";
+import axios from "axios";
+import OpenAI from "openai";
+import { z } from "zod";
+import { zodResponseFormat } from "openai/helpers/zod";
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { Blob } from "buffer";
+import { FastifyInstance } from "fastify";
+
+import { s3 } from "./s3";
+import { secrets } from "./secrets";
+
+// Drizzle schema
+import { vendors, accounts } from "../../db/schema";
+import {eq} from "drizzle-orm";
+
+let openai: OpenAI | null = null;
+
+// ---------------------------------------------------------
+// INITIALIZE OPENAI
+// ---------------------------------------------------------
+export const initOpenAi = async () => {
+    openai = new OpenAI({
+        apiKey: secrets.OPENAI_API_KEY,
+    });
+};
+
+// ---------------------------------------------------------
+// STREAM → BUFFER
+// ---------------------------------------------------------
+async function streamToBuffer(stream: any): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+        const chunks: Buffer[] = [];
+        stream.on("data", (chunk: Buffer) => chunks.push(chunk));
+        stream.on("error", reject);
+        stream.on("end", () => resolve(Buffer.concat(chunks)));
+    });
+}
+
+// ---------------------------------------------------------
+// GPT RESPONSE FORMAT (Zod Schema)
+// ---------------------------------------------------------
+const InstructionFormat = z.object({
+    invoice_number: z.string(),
+    invoice_date: z.string(),
+    invoice_duedate: z.string(),
+    invoice_type: z.string(),
+    delivery_type: z.string(),
+    delivery_note_number: z.string(),
+    reference: z.string(),
+    issuer: z.object({
+        id: z.number().nullable().optional(),
+        name: z.string(),
+        address: z.string(),
+        phone: z.string(),
+        email: z.string(),
+        bank: z.string(),
+        bic: z.string(),
+        iban: z.string(),
+    }),
+    recipient: z.object({
+        name: z.string(),
+        address: z.string(),
+        phone: z.string(),
+        email: z.string(),
+    }),
+    invoice_items: z.array(
+        z.object({
+            description: z.string(),
+            unit: z.string(),
+            quantity: z.number(),
+            total: z.number(),
+            total_without_tax: z.number(),
+            tax_rate: z.number(),
+            ean: z.number().nullable().optional(),
+            article_number: z.number().nullable().optional(),
+            account_number: z.number().nullable().optional(),
+            account_id: z.number().nullable().optional(),
+        })
+    ),
+    subtotal: z.number(),
+    tax_rate: z.number(),
+    tax: z.number(),
+    total: z.number(),
+    terms: z.string(),
+});
+
+// ---------------------------------------------------------
+// MAIN FUNCTION – REPLACES SUPABASE VERSION
+// ---------------------------------------------------------
+export const getInvoiceDataFromGPT = async function (
+    server: FastifyInstance,
+    file: any,
+    tenantId: number
+) {
+    await initOpenAi();
+
+    if (!openai) {
+        throw new Error("OpenAI not initialized. Call initOpenAi() first.");
+    }
+
+    console.log(`📄 Reading invoice file ${file.id}`);
+
+    // ---------------------------------------------------------
+    // 1) DOWNLOAD PDF FROM S3
+    // ---------------------------------------------------------
+    let fileData: Buffer;
+
+    try {
+        const command = new GetObjectCommand({
+            Bucket: secrets.S3_BUCKET,
+            Key: file.path,
+        });
+
+        const response: any = await s3.send(command);
+        fileData = await streamToBuffer(response.Body);
+    } catch (err) {
+        console.log(`❌ S3 Download failed for file ${file.id}`, err);
+        return null;
+    }
+
+    // Only process PDFs
+    if (!file.path.toLowerCase().endsWith(".pdf")) {
+        server.log.warn(`Skipping non-PDF file ${file.id}`);
+        return null;
+    }
+
+    const fileBlob = new Blob([fileData], { type: "application/pdf" });
+
+    // ---------------------------------------------------------
+    // 2) SEND FILE TO PDF → TEXT API
+    // ---------------------------------------------------------
+    const form = new FormData();
+    form.append("fileInput", fileBlob, file.path.split("/").pop());
+    form.append("outputFormat", "txt");
+
+    let extractedText: string;
+
+    try {
+        const res = await axios.post(
+            "http://23.88.52.85:8080/api/v1/convert/pdf/text",
+            form,
+            {
+                headers: {
+                    "Content-Type": "multipart/form-data",
+                    Authorization: `Bearer ${secrets.STIRLING_API_KEY}`,
+                },
+            }
+        );
+
+        extractedText = res.data;
+    } catch (err) {
+        console.log("❌ PDF OCR API failed", err);
+        return null;
+    }
+
+    // ---------------------------------------------------------
+    // 3) LOAD VENDORS + ACCOUNTS (DRIZZLE)
+    // ---------------------------------------------------------
+    const vendorList = await server.db
+        .select({ id: vendors.id, name: vendors.name })
+        .from(vendors)
+        .where(eq(vendors.tenant,tenantId));
+
+    const accountList = await server.db
+        .select({
+            id: accounts.id,
+            label: accounts.label,
+            number: accounts.number,
+        })
+        .from(accounts);
+
+    // ---------------------------------------------------------
+    // 4) GPT ANALYSIS
+    // ---------------------------------------------------------
+
+
+
+    const completion = await openai.chat.completions.parse({
+        model: "gpt-4o",
+        store: true,
+        response_format: zodResponseFormat(InstructionFormat as any, "instruction"),
+        messages: [
+            { role: "user", content: extractedText },
+            {
+                role: "user",
+                content:
+                    "You extract structured invoice data.\n\n" +
+                    `VENDORS: ${JSON.stringify(vendorList)}\n` +
+                    `ACCOUNTS: ${JSON.stringify(accountList)}\n\n` +
+                    "Match issuer by name to vendor.id.\n" +
+                    "Match invoice items to account id based on label/number.\n" +
+                    "Convert dates to YYYY-MM-DD.\n" +
+                    "Keep invoice items in original order.\n",
+            },
+        ],
+    });
+
+    const parsed = completion.choices[0].message.parsed;
+
+    console.log(`🧾 Extracted invoice data for file ${file.id}`);
+
+    return parsed;
+};

src/utils/secrets.ts

@@ -33,6 +33,13 @@ export let secrets = {
     GOCARDLESS_BASE_URL: string
     GOCARDLESS_SECRET_ID: string
     GOCARDLESS_SECRET_KEY: string
+    DOKUBOX_IMAP_HOST: string
+    DOKUBOX_IMAP_PORT: number
+    DOKUBOX_IMAP_SECURE: boolean
+    DOKUBOX_IMAP_USER: string
+    DOKUBOX_IMAP_PASSWORD: string
+    OPENAI_API_KEY: string
+    STIRLING_API_KEY: string
 }
 
 export async function loadSecrets () {