Fixes

backend/src/routes/functions.ts

@@ -1,6 +1,7 @@
 import { FastifyInstance } from "fastify";
 import {createInvoicePDF, createTimeSheetPDF} from "../utils/pdf";
 import {encodeBase64ToNiimbot, generateLabel, useNextNumberRangeNumber} from "../utils/functions";
+import { GetObjectCommand } from "@aws-sdk/client-s3";
 import dayjs from "dayjs";
 //import { ready as zplReady } from 'zpl-renderer-js'
 //import { renderZPL } from "zpl-image";
@@ -13,9 +14,12 @@ import isSameOrBefore from "dayjs/plugin/isSameOrBefore.js"
 import duration from "dayjs/plugin/duration.js";
 import timezone from "dayjs/plugin/timezone.js";
 import {generateTimesEvaluation} from "../modules/time/evaluation.service";
-import {citys} from "../../db/schema";
-import {eq} from "drizzle-orm";
+import {citys, files} from "../../db/schema";
+import {and, eq, isNull, not} from "drizzle-orm";
 import {executeManualGeneration, finishManualGeneration} from "../modules/serialexecution.service";
+import { s3 } from "../utils/s3";
+import { secrets } from "../utils/secrets";
+import { storeExtractedTextForFile } from "../utils/documentText";
 dayjs.extend(customParseFormat)
 dayjs.extend(isoWeek)
 dayjs.extend(isBetween)
@@ -25,6 +29,14 @@ dayjs.extend(duration)
 dayjs.extend(timezone)
 
 export default async function functionRoutes(server: FastifyInstance) {
+    const streamToBuffer = async (stream: any): Promise<Buffer> =>
+        new Promise((resolve, reject) => {
+            const chunks: Buffer[] = [];
+            stream.on("data", (chunk: Buffer) => chunks.push(chunk));
+            stream.on("error", reject);
+            stream.on("end", () => resolve(Buffer.concat(chunks)));
+        });
+
     server.post("/functions/pdf/:type", async (req, reply) => {
         const body = req.body as {
             data: any
@@ -171,6 +183,58 @@ export default async function functionRoutes(server: FastifyInstance) {
         await server.services.prepareIncomingInvoices.run(req.user.tenant_id)
     })
 
+    server.post('/functions/services/backfillfiletext', async (req, reply) => {
+        const tenantId = req.user.tenant_id
+
+        const pendingFiles = await server.db
+            .select()
+            .from(files)
+            .where(
+                and(
+                    eq(files.tenant, tenantId),
+                    eq(files.archived, false),
+                    not(isNull(files.path)),
+                    isNull(files.extractedText)
+                )
+            )
+
+        let processed = 0
+        let withText = 0
+        let errors = 0
+
+        for (const file of pendingFiles) {
+            try {
+                const response: any = await s3.send(new GetObjectCommand({
+                    Bucket: secrets.S3_BUCKET,
+                    Key: file.path!
+                }))
+
+                const fileBuffer = await streamToBuffer(response.Body)
+                const result = await storeExtractedTextForFile(
+                    server,
+                    file.id,
+                    fileBuffer,
+                    file.mimeType,
+                    file.name || file.path?.split("/").pop()
+                )
+
+                processed += 1
+                if (result.text) withText += 1
+            } catch (err) {
+                errors += 1
+                server.log.error(`Failed to backfill extracted text for file ${file.id}`)
+                server.log.error(err)
+            }
+        }
+
+        return {
+            pending: pendingFiles.length,
+            processed,
+            withText,
+            errors
+        }
+    })
+
     server.post('/functions/services/syncdokubox', async (req, reply) => {
 
         await server.services.dokuboxSync.run()