Fix tenant archive S3 upload

backend/src/utils/tenantFullExport.ts

@@ -1,7 +1,10 @@
 import { FastifyInstance } from "fastify"
 import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
 import archiver from "archiver"
-import { PassThrough } from "stream"
+import { createReadStream, createWriteStream } from "fs"
+import { stat, unlink } from "fs/promises"
+import os from "os"
+import path from "path"
 import { BlobReader, TextWriter, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js"
 
 import { pool } from "../../db"
@@ -382,89 +385,100 @@ export const createTenantFullExportArchive = async (
     const storagePath = options.storagePath || `tenant-exports/${tenantId}/${Date.now()}-${filename}`
     const fileRows = exportData.tables.files || []
     const archive = archiver("zip", { zlib: { level: 6 } })
-    const output = new PassThrough()
+    const tempPath = path.join(os.tmpdir(), `fedeo-tenant-export-${tenantId}-${Date.now()}-${filename}`)
-    let size = 0
+    const output = createWriteStream(tempPath)
-
+    const archiveComplete = new Promise<void>((resolve, reject) => {
-    output.on("data", (chunk: Buffer) => {
+        output.on("close", resolve)
-        size += chunk.length
+        output.on("error", reject)
+        archive.on("error", reject)
+        archive.on("warning", (err) => {
+            if ((err as NodeJS.ErrnoException).code === "ENOENT") return
+            reject(err)
+        })
     })
 
-    const uploadPromise = s3.send(new PutObjectCommand({
-        Bucket: secrets.S3_BUCKET,
-        Key: storagePath,
-        Body: output,
-        ContentType: "application/zip",
-        ContentDisposition: `attachment; filename="${filename}"`,
-    }))
-
     archive.pipe(output)
 
-    const tables = Object.entries(exportData.tables)
+    try {
-        .sort(([a], [b]) => a.localeCompare(b))
+        const tables = Object.entries(exportData.tables)
-        .map(([table, rows]) => {
+            .sort(([a], [b]) => a.localeCompare(b))
-            const path = `tables/${table}.json`
+            .map(([table, rows]) => {
-            archive.append(jsonBuffer(rows), { name: path })
+                const path = `tables/${table}.json`
-            return { name: table, rows: rows.length, path }
+                archive.append(jsonBuffer(rows), { name: path })
-        })
+                return { name: table, rows: rows.length, path }
+            })
 
-    const manifestFiles: TenantArchiveManifest["files"] = []
+        const manifestFiles: TenantArchiveManifest["files"] = []
-    let filesDone = 0
+        let filesDone = 0
-    const filesTotal = fileRows.filter((file) => file.path).length
+        const filesTotal = fileRows.filter((file) => file.path).length
 
-    for (const file of fileRows) {
+        for (const file of fileRows) {
-        if (!file.path) continue
+            if (!file.path) continue
 
-        const archivePath = archiveEntryPathForFile(file.path)
+            const archivePath = archiveEntryPathForFile(file.path)
-        const object = await loadObjectStream(file.path)
+            const object = await loadObjectStream(file.path)
 
-        if (object.missing) {
+            if (object.missing) {
-            server.log.warn({
+                server.log.warn({
-                fileId: file.id,
+                    fileId: file.id,
+                    path: file.path,
+                    bucket: secrets.S3_BUCKET,
+                    error: object.error,
+                }, "Tenant archive export skipped missing S3 object")
+            } else {
+                archive.append(object.body, { name: archivePath })
+            }
+
+            filesDone += 1
+            manifestFiles.push({
+                id: file.id,
                 path: file.path,
-                bucket: secrets.S3_BUCKET,
+                archivePath: object.missing ? null : archivePath,
-                error: object.error,
+                name: file.name || null,
-            }, "Tenant archive export skipped missing S3 object")
+                mimeType: file.mimeType || null,
-        } else {
+                size: file.size || null,
-            archive.append(object.body, { name: archivePath })
+                missing: object.missing || undefined,
+                error: object.error || undefined,
+            })
+
+            if (filesDone % 10 === 0 || filesDone === filesTotal) {
+                await options.onProgress?.({ filesDone, filesTotal })
+            }
         }
 
-        filesDone += 1
+        const manifest: TenantArchiveManifest = {
-        manifestFiles.push({
+            format: "fedeo.tenant-archive-export",
-            id: file.id,
+            version: 1,
-            path: file.path,
+            exportedAt: exportData.exportedAt,
-            archivePath: object.missing ? null : archivePath,
+            tenantId,
-            name: file.name || null,
+            tables,
-            mimeType: file.mimeType || null,
+            files: manifestFiles,
-            size: file.size || null,
-            missing: object.missing || undefined,
-            error: object.error || undefined,
-        })
-
-        if (filesDone % 10 === 0 || filesDone === filesTotal) {
-            await options.onProgress?.({ filesDone, filesTotal })
         }
-    }
 
-    const manifest: TenantArchiveManifest = {
+        archive.append(jsonBuffer(manifest), { name: "manifest.json" })
-        format: "fedeo.tenant-archive-export",
+        await archive.finalize()
-        version: 1,
+        await archiveComplete
-        exportedAt: exportData.exportedAt,
-        tenantId,
-        tables,
-        files: manifestFiles,
-    }
 
-    archive.append(jsonBuffer(manifest), { name: "manifest.json" })
+        const { size } = await stat(tempPath)
-    await archive.finalize()
+        await s3.send(new PutObjectCommand({
-    await uploadPromise
+            Bucket: secrets.S3_BUCKET,
+            Key: storagePath,
+            Body: createReadStream(tempPath),
+            ContentLength: size,
+            ContentType: "application/zip",
+            ContentDisposition: `attachment; filename="${filename}"`,
+        }))
 
-    return {
+        return {
-        filename,
+            filename,
-        storagePath,
+            storagePath,
-        contentType: "application/zip",
+            contentType: "application/zip",
-        size,
+            size,
-        filesTotal,
+            filesTotal,
-        filesDone,
+            filesDone,
+        }
+    } finally {
+        await unlink(tempPath).catch(() => undefined)
     }
 }