@@ -42,6 +42,7 @@
|
||||
"imapflow": "^1.1.1",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"nodemailer": "^7.0.6",
|
||||
"openai": "^6.10.0",
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pg": "^8.16.3",
|
||||
"pngjs": "^7.0.0",
|
||||
|
||||
175
src/modules/cron/prepareIncomingInvoices.ts
Normal file
175
src/modules/cron/prepareIncomingInvoices.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
import { FastifyInstance } from "fastify"
|
||||
import dayjs from "dayjs"
|
||||
import { getInvoiceDataFromGPT } from "../../utils/gpt"
|
||||
|
||||
// Drizzle schema
|
||||
import {
|
||||
tenants,
|
||||
files,
|
||||
filetags,
|
||||
incominginvoices,
|
||||
} from "../../../db/schema"
|
||||
|
||||
import { eq, and, isNull, not } from "drizzle-orm"
|
||||
|
||||
export function prepareIncomingInvoices(server: FastifyInstance) {
|
||||
const processInvoices = async (tenantId:number) => {
|
||||
console.log("▶ Starting Incoming Invoice Preparation")
|
||||
|
||||
const tenantsRes = await server.db
|
||||
.select()
|
||||
.from(tenants)
|
||||
.where(eq(tenants.id, tenantId))
|
||||
.orderBy(tenants.id)
|
||||
|
||||
if (!tenantsRes.length) {
|
||||
console.log("No tenants with autoPrepareIncomingInvoices = true")
|
||||
return
|
||||
}
|
||||
|
||||
console.log(`Processing tenants: ${tenantsRes.map(t => t.id).join(", ")}`)
|
||||
|
||||
// -------------------------------------------------------------
|
||||
// 2️⃣ Jeden Tenant einzeln verarbeiten
|
||||
// -------------------------------------------------------------
|
||||
for (const tenant of tenantsRes) {
|
||||
const tenantId = tenant.id
|
||||
|
||||
// 2.1 Datei-Tags holen für incoming invoices
|
||||
const tagRes = await server.db
|
||||
.select()
|
||||
.from(filetags)
|
||||
.where(
|
||||
and(
|
||||
eq(filetags.tenant, tenantId),
|
||||
eq(filetags.incomingDocumentType, "invoices")
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
|
||||
const invoiceFileTag = tagRes?.[0]?.id
|
||||
if (!invoiceFileTag) {
|
||||
server.log.error(`❌ Missing filetag 'invoices' for tenant ${tenantId}`)
|
||||
continue
|
||||
}
|
||||
|
||||
// 2.2 Alle Dateien laden, die als Invoice markiert sind aber NOCH keine incominginvoice haben
|
||||
const filesRes = await server.db
|
||||
.select()
|
||||
.from(files)
|
||||
.where(
|
||||
and(
|
||||
eq(files.tenant, tenantId),
|
||||
eq(files.type, invoiceFileTag),
|
||||
isNull(files.incominginvoice),
|
||||
eq(files.archived, false),
|
||||
not(isNull(files.path))
|
||||
)
|
||||
)
|
||||
|
||||
if (!filesRes.length) {
|
||||
console.log(`No invoice files for tenant ${tenantId}`)
|
||||
continue
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------
|
||||
// 3️⃣ Jede Datei einzeln durch GPT jagen & IncomingInvoice erzeugen
|
||||
// -------------------------------------------------------------
|
||||
for (const file of filesRes) {
|
||||
console.log(`Processing file ${file.id} for tenant ${tenantId}`)
|
||||
|
||||
const data = await getInvoiceDataFromGPT(server,file, tenantId)
|
||||
|
||||
if (!data) {
|
||||
server.log.warn(`GPT returned no data for file ${file.id}`)
|
||||
continue
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 3.1 IncomingInvoice-Objekt vorbereiten
|
||||
// ---------------------------------------------------------
|
||||
let itemInfo: any = {
|
||||
tenant: tenantId,
|
||||
state: "Vorbereitet"
|
||||
}
|
||||
|
||||
if (data.invoice_number) itemInfo.reference = data.invoice_number
|
||||
if (data.invoice_date) itemInfo.date = dayjs(data.invoice_date).toISOString()
|
||||
if (data.issuer?.id) itemInfo.vendor = data.issuer.id
|
||||
if (data.invoice_duedate) itemInfo.dueDate = dayjs(data.invoice_duedate).toISOString()
|
||||
|
||||
// Payment terms mapping
|
||||
const mapPayment: any = {
|
||||
"Direct Debit": "Einzug",
|
||||
"Transfer": "Überweisung",
|
||||
"Credit Card": "Kreditkarte",
|
||||
"Other": "Sonstiges",
|
||||
}
|
||||
if (data.terms) itemInfo.paymentType = mapPayment[data.terms] ?? data.terms
|
||||
|
||||
// 3.2 Positionszeilen konvertieren
|
||||
if (data.invoice_items?.length > 0) {
|
||||
itemInfo.accounts = data.invoice_items.map(item => ({
|
||||
account: item.account_id,
|
||||
description: item.description,
|
||||
amountNet: item.total_without_tax,
|
||||
amountTax: Number((item.total - item.total_without_tax).toFixed(2)),
|
||||
taxType: String(item.tax_rate),
|
||||
amountGross: item.total,
|
||||
costCentre: null,
|
||||
quantity: item.quantity,
|
||||
}))
|
||||
}
|
||||
|
||||
// 3.3 Beschreibung generieren
|
||||
let description = ""
|
||||
if (data.delivery_note_number) description += `Lieferschein: ${data.delivery_note_number}\n`
|
||||
if (data.reference) description += `Referenz: ${data.reference}\n`
|
||||
if (data.invoice_items) {
|
||||
for (const item of data.invoice_items) {
|
||||
description += `${item.description} - ${item.quantity} ${item.unit} - ${item.total}\n`
|
||||
}
|
||||
}
|
||||
itemInfo.description = description.trim()
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 4️⃣ IncomingInvoice erstellen
|
||||
// ---------------------------------------------------------
|
||||
const inserted = await server.db
|
||||
.insert(incominginvoices)
|
||||
.values(itemInfo)
|
||||
.returning()
|
||||
|
||||
const newInvoice = inserted?.[0]
|
||||
|
||||
if (!newInvoice) {
|
||||
server.log.error(`Failed to insert incoming invoice for file ${file.id}`)
|
||||
continue
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 5️⃣ Datei mit incominginvoice-ID verbinden
|
||||
// ---------------------------------------------------------
|
||||
await server.db
|
||||
.update(files)
|
||||
.set({ incominginvoice: newInvoice.id })
|
||||
.where(eq(files.id, file.id))
|
||||
|
||||
console.log(`IncomingInvoice ${newInvoice.id} created for file ${file.id}`)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
return {
|
||||
run: async (tenant:number) => {
|
||||
await processInvoices(tenant)
|
||||
console.log("Incoming Invoice Preparation Completed.")
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -388,15 +388,19 @@ async function getCloseData(server:FastifyInstance,item: any, tenant: any, units
|
||||
|
||||
console.log(item);
|
||||
|
||||
const [contact, customer, profile, project, contract] = await Promise.all([
|
||||
const [contact, customer, project, contract] = await Promise.all([
|
||||
fetchById(server, schema.contacts, item.contact),
|
||||
fetchById(server, schema.customers, item.customer),
|
||||
fetchById(server, schema.authProfiles, item.contactPerson), // oder createdBy, je nach Logik
|
||||
fetchById(server, schema.projects, item.project),
|
||||
fetchById(server, schema.contracts, item.contract),
|
||||
item.letterhead ? fetchById(server, schema.letterheads, item.letterhead) : null
|
||||
|
||||
]);
|
||||
|
||||
const profile = (await server.db.select().from(schema.authProfiles).where(and(eq(schema.authProfiles.user_id, item.created_by),eq(schema.authProfiles.tenant_id,tenant.id))).limit(1))[0];
|
||||
|
||||
console.log(profile)
|
||||
|
||||
const pdfData = getDocumentDataBackend(
|
||||
{
|
||||
...item,
|
||||
@@ -613,6 +617,8 @@ export function getDocumentDataBackend(
|
||||
`${itemInfo.address?.zip || customerData.zip} ${itemInfo.address?.city || customerData.city}`,
|
||||
].filter(Boolean); // Leere Einträge entfernen
|
||||
|
||||
console.log(contactPerson)
|
||||
|
||||
// Info Block aufbereiten
|
||||
const infoBlock = [
|
||||
{
|
||||
@@ -634,12 +640,12 @@ export function getDocumentDataBackend(
|
||||
}] : []),
|
||||
{
|
||||
label: "Ansprechpartner",
|
||||
content: contactPerson ? (contactPerson.name || contactPerson.fullName || contactPerson.email) : "-",
|
||||
content: contactPerson ? (contactPerson.name || contactPerson.full_name || contactPerson.email) : "-",
|
||||
},
|
||||
// Kontakt Infos
|
||||
...((itemInfo.contactTel || contactPerson?.fixedTel || contactPerson?.mobileTel) ? [{
|
||||
...((itemInfo.contactTel || contactPerson?.fixed_tel || contactPerson?.mobile_tel) ? [{
|
||||
label: "Telefon",
|
||||
content: itemInfo.contactTel || contactPerson?.fixedTel || contactPerson?.mobileTel,
|
||||
content: itemInfo.contactTel || contactPerson?.fixed_tel || contactPerson?.mobile_tel,
|
||||
}] : []),
|
||||
...(contactPerson?.email ? [{
|
||||
label: "E-Mail",
|
||||
|
||||
@@ -3,14 +3,14 @@ import fp from "fastify-plugin";
|
||||
import { bankStatementService } from "../modules/cron/bankstatementsync.service";
|
||||
//import {initDokuboxClient, syncDokubox} from "../modules/cron/dokuboximport.service";
|
||||
import { FastifyInstance } from "fastify";
|
||||
//import {prepareIncomingInvoices} from "../modules/cron/prepareIncomingInvoices";
|
||||
import {prepareIncomingInvoices} from "../modules/cron/prepareIncomingInvoices";
|
||||
|
||||
declare module "fastify" {
|
||||
interface FastifyInstance {
|
||||
services: {
|
||||
bankStatements: ReturnType<typeof bankStatementService>;
|
||||
//dokuboxSync: ReturnType<typeof syncDokubox>;
|
||||
//prepareIncomingInvoices: ReturnType<typeof prepareIncomingInvoices>;
|
||||
prepareIncomingInvoices: ReturnType<typeof prepareIncomingInvoices>;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,6 @@ export default fp(async function servicePlugin(server: FastifyInstance) {
|
||||
server.decorate("services", {
|
||||
bankStatements: bankStatementService(server),
|
||||
//dokuboxSync: syncDokubox(server),
|
||||
//prepareIncomingInvoices: prepareIncomingInvoices(server),
|
||||
prepareIncomingInvoices: prepareIncomingInvoices(server),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -174,6 +174,11 @@ export default async function functionRoutes(server: FastifyInstance) {
|
||||
await server.services.bankStatements.run(req.user.tenant_id);
|
||||
})
|
||||
|
||||
server.post('/functions/services/prepareincominginvoices', async (req, reply) => {
|
||||
|
||||
await server.services.prepareIncomingInvoices.run(req.user.tenant_id)
|
||||
})
|
||||
|
||||
|
||||
/*server.post('/print/zpl/preview', async (req, reply) => {
|
||||
const { zpl, widthMm = 50, heightMm = 30, dpmm = 8, asBase64 = false } = req.body as {zpl:string,widthMm:number,heightMm:number,dpmm:number,asBase64:string}
|
||||
|
||||
204
src/utils/gpt.ts
Normal file
204
src/utils/gpt.ts
Normal file
@@ -0,0 +1,204 @@
|
||||
import dayjs from "dayjs";
|
||||
import axios from "axios";
|
||||
import OpenAI from "openai";
|
||||
import { z } from "zod";
|
||||
import { zodResponseFormat } from "openai/helpers/zod";
|
||||
import { GetObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { Blob } from "buffer";
|
||||
import { FastifyInstance } from "fastify";
|
||||
|
||||
import { s3 } from "./s3";
|
||||
import { secrets } from "./secrets";
|
||||
|
||||
// Drizzle schema
|
||||
import { vendors, accounts } from "../../db/schema";
|
||||
import {eq} from "drizzle-orm";
|
||||
|
||||
let openai: OpenAI | null = null;
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// INITIALIZE OPENAI
|
||||
// ---------------------------------------------------------
|
||||
export const initOpenAi = async () => {
|
||||
openai = new OpenAI({
|
||||
apiKey: secrets.OPENAI_API_KEY,
|
||||
});
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// STREAM → BUFFER
|
||||
// ---------------------------------------------------------
|
||||
async function streamToBuffer(stream: any): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
stream.on("error", reject);
|
||||
stream.on("end", () => resolve(Buffer.concat(chunks)));
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// GPT RESPONSE FORMAT (Zod Schema)
|
||||
// ---------------------------------------------------------
|
||||
const InstructionFormat = z.object({
|
||||
invoice_number: z.string(),
|
||||
invoice_date: z.string(),
|
||||
invoice_duedate: z.string(),
|
||||
invoice_type: z.string(),
|
||||
delivery_type: z.string(),
|
||||
delivery_note_number: z.string(),
|
||||
reference: z.string(),
|
||||
issuer: z.object({
|
||||
id: z.number().nullable().optional(),
|
||||
name: z.string(),
|
||||
address: z.string(),
|
||||
phone: z.string(),
|
||||
email: z.string(),
|
||||
bank: z.string(),
|
||||
bic: z.string(),
|
||||
iban: z.string(),
|
||||
}),
|
||||
recipient: z.object({
|
||||
name: z.string(),
|
||||
address: z.string(),
|
||||
phone: z.string(),
|
||||
email: z.string(),
|
||||
}),
|
||||
invoice_items: z.array(
|
||||
z.object({
|
||||
description: z.string(),
|
||||
unit: z.string(),
|
||||
quantity: z.number(),
|
||||
total: z.number(),
|
||||
total_without_tax: z.number(),
|
||||
tax_rate: z.number(),
|
||||
ean: z.number().nullable().optional(),
|
||||
article_number: z.number().nullable().optional(),
|
||||
account_number: z.number().nullable().optional(),
|
||||
account_id: z.number().nullable().optional(),
|
||||
})
|
||||
),
|
||||
subtotal: z.number(),
|
||||
tax_rate: z.number(),
|
||||
tax: z.number(),
|
||||
total: z.number(),
|
||||
terms: z.string(),
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// MAIN FUNCTION – REPLACES SUPABASE VERSION
|
||||
// ---------------------------------------------------------
|
||||
export const getInvoiceDataFromGPT = async function (
|
||||
server: FastifyInstance,
|
||||
file: any,
|
||||
tenantId: number
|
||||
) {
|
||||
await initOpenAi();
|
||||
|
||||
if (!openai) {
|
||||
throw new Error("OpenAI not initialized. Call initOpenAi() first.");
|
||||
}
|
||||
|
||||
console.log(`📄 Reading invoice file ${file.id}`);
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 1) DOWNLOAD PDF FROM S3
|
||||
// ---------------------------------------------------------
|
||||
let fileData: Buffer;
|
||||
|
||||
try {
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: secrets.S3_BUCKET,
|
||||
Key: file.path,
|
||||
});
|
||||
|
||||
const response: any = await s3.send(command);
|
||||
fileData = await streamToBuffer(response.Body);
|
||||
} catch (err) {
|
||||
console.log(`❌ S3 Download failed for file ${file.id}`, err);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Only process PDFs
|
||||
if (!file.path.toLowerCase().endsWith(".pdf")) {
|
||||
server.log.warn(`Skipping non-PDF file ${file.id}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const fileBlob = new Blob([fileData], { type: "application/pdf" });
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 2) SEND FILE TO PDF → TEXT API
|
||||
// ---------------------------------------------------------
|
||||
const form = new FormData();
|
||||
form.append("fileInput", fileBlob, file.path.split("/").pop());
|
||||
form.append("outputFormat", "txt");
|
||||
|
||||
let extractedText: string;
|
||||
|
||||
try {
|
||||
const res = await axios.post(
|
||||
"http://23.88.52.85:8080/api/v1/convert/pdf/text",
|
||||
form,
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "multipart/form-data",
|
||||
Authorization: `Bearer ${secrets.STIRLING_API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
extractedText = res.data;
|
||||
} catch (err) {
|
||||
console.log("❌ PDF OCR API failed", err);
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 3) LOAD VENDORS + ACCOUNTS (DRIZZLE)
|
||||
// ---------------------------------------------------------
|
||||
const vendorList = await server.db
|
||||
.select({ id: vendors.id, name: vendors.name })
|
||||
.from(vendors)
|
||||
.where(eq(vendors.tenant,tenantId));
|
||||
|
||||
const accountList = await server.db
|
||||
.select({
|
||||
id: accounts.id,
|
||||
label: accounts.label,
|
||||
number: accounts.number,
|
||||
})
|
||||
.from(accounts);
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 4) GPT ANALYSIS
|
||||
// ---------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
const completion = await openai.chat.completions.parse({
|
||||
model: "gpt-4o",
|
||||
store: true,
|
||||
response_format: zodResponseFormat(InstructionFormat as any, "instruction"),
|
||||
messages: [
|
||||
{ role: "user", content: extractedText },
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"You extract structured invoice data.\n\n" +
|
||||
`VENDORS: ${JSON.stringify(vendorList)}\n` +
|
||||
`ACCOUNTS: ${JSON.stringify(accountList)}\n\n` +
|
||||
"Match issuer by name to vendor.id.\n" +
|
||||
"Match invoice items to account id based on label/number.\n" +
|
||||
"Convert dates to YYYY-MM-DD.\n" +
|
||||
"Keep invoice items in original order.\n",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const parsed = completion.choices[0].message.parsed;
|
||||
|
||||
console.log(`🧾 Extracted invoice data for file ${file.id}`);
|
||||
|
||||
return parsed;
|
||||
};
|
||||
@@ -33,6 +33,13 @@ export let secrets = {
|
||||
GOCARDLESS_BASE_URL: string
|
||||
GOCARDLESS_SECRET_ID: string
|
||||
GOCARDLESS_SECRET_KEY: string
|
||||
DOKUBOX_IMAP_HOST: string
|
||||
DOKUBOX_IMAP_PORT: number
|
||||
DOKUBOX_IMAP_SECURE: boolean
|
||||
DOKUBOX_IMAP_USER: string
|
||||
DOKUBOX_IMAP_PASSWORD: string
|
||||
OPENAI_API_KEY: string
|
||||
STIRLING_API_KEY: string
|
||||
}
|
||||
|
||||
export async function loadSecrets () {
|
||||
|
||||
Reference in New Issue
Block a user