diff --git a/backend/src/utils/tenantFullExport.ts b/backend/src/utils/tenantFullExport.ts index 5a0ec2b..43a48e3 100644 --- a/backend/src/utils/tenantFullExport.ts +++ b/backend/src/utils/tenantFullExport.ts @@ -1,7 +1,10 @@ import { FastifyInstance } from "fastify" import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3" import archiver from "archiver" -import { PassThrough } from "stream" +import { createReadStream, createWriteStream } from "fs" +import { stat, unlink } from "fs/promises" +import os from "os" +import path from "path" import { BlobReader, TextWriter, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js" import { pool } from "../../db" @@ -382,89 +385,100 @@ export const createTenantFullExportArchive = async ( const storagePath = options.storagePath || `tenant-exports/${tenantId}/${Date.now()}-${filename}` const fileRows = exportData.tables.files || [] const archive = archiver("zip", { zlib: { level: 6 } }) - const output = new PassThrough() - let size = 0 - - output.on("data", (chunk: Buffer) => { - size += chunk.length + const tempPath = path.join(os.tmpdir(), `fedeo-tenant-export-${tenantId}-${Date.now()}-${filename}`) + const output = createWriteStream(tempPath) + const archiveComplete = new Promise((resolve, reject) => { + output.on("close", resolve) + output.on("error", reject) + archive.on("error", reject) + archive.on("warning", (err) => { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return + reject(err) + }) }) - const uploadPromise = s3.send(new PutObjectCommand({ - Bucket: secrets.S3_BUCKET, - Key: storagePath, - Body: output, - ContentType: "application/zip", - ContentDisposition: `attachment; filename="${filename}"`, - })) - archive.pipe(output) - const tables = Object.entries(exportData.tables) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([table, rows]) => { - const path = `tables/${table}.json` - archive.append(jsonBuffer(rows), { name: path }) - return { name: table, rows: rows.length, path } - }) + try { + const tables = Object.entries(exportData.tables) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([table, rows]) => { + const path = `tables/${table}.json` + archive.append(jsonBuffer(rows), { name: path }) + return { name: table, rows: rows.length, path } + }) - const manifestFiles: TenantArchiveManifest["files"] = [] - let filesDone = 0 - const filesTotal = fileRows.filter((file) => file.path).length + const manifestFiles: TenantArchiveManifest["files"] = [] + let filesDone = 0 + const filesTotal = fileRows.filter((file) => file.path).length - for (const file of fileRows) { - if (!file.path) continue + for (const file of fileRows) { + if (!file.path) continue - const archivePath = archiveEntryPathForFile(file.path) - const object = await loadObjectStream(file.path) + const archivePath = archiveEntryPathForFile(file.path) + const object = await loadObjectStream(file.path) - if (object.missing) { - server.log.warn({ - fileId: file.id, + if (object.missing) { + server.log.warn({ + fileId: file.id, + path: file.path, + bucket: secrets.S3_BUCKET, + error: object.error, + }, "Tenant archive export skipped missing S3 object") + } else { + archive.append(object.body, { name: archivePath }) + } + + filesDone += 1 + manifestFiles.push({ + id: file.id, path: file.path, - bucket: secrets.S3_BUCKET, - error: object.error, - }, "Tenant archive export skipped missing S3 object") - } else { - archive.append(object.body, { name: archivePath }) + archivePath: object.missing ? null : archivePath, + name: file.name || null, + mimeType: file.mimeType || null, + size: file.size || null, + missing: object.missing || undefined, + error: object.error || undefined, + }) + + if (filesDone % 10 === 0 || filesDone === filesTotal) { + await options.onProgress?.({ filesDone, filesTotal }) + } } - filesDone += 1 - manifestFiles.push({ - id: file.id, - path: file.path, - archivePath: object.missing ? null : archivePath, - name: file.name || null, - mimeType: file.mimeType || null, - size: file.size || null, - missing: object.missing || undefined, - error: object.error || undefined, - }) - - if (filesDone % 10 === 0 || filesDone === filesTotal) { - await options.onProgress?.({ filesDone, filesTotal }) + const manifest: TenantArchiveManifest = { + format: "fedeo.tenant-archive-export", + version: 1, + exportedAt: exportData.exportedAt, + tenantId, + tables, + files: manifestFiles, } - } - const manifest: TenantArchiveManifest = { - format: "fedeo.tenant-archive-export", - version: 1, - exportedAt: exportData.exportedAt, - tenantId, - tables, - files: manifestFiles, - } + archive.append(jsonBuffer(manifest), { name: "manifest.json" }) + await archive.finalize() + await archiveComplete - archive.append(jsonBuffer(manifest), { name: "manifest.json" }) - await archive.finalize() - await uploadPromise + const { size } = await stat(tempPath) + await s3.send(new PutObjectCommand({ + Bucket: secrets.S3_BUCKET, + Key: storagePath, + Body: createReadStream(tempPath), + ContentLength: size, + ContentType: "application/zip", + ContentDisposition: `attachment; filename="${filename}"`, + })) - return { - filename, - storagePath, - contentType: "application/zip", - size, - filesTotal, - filesDone, + return { + filename, + storagePath, + contentType: "application/zip", + size, + filesTotal, + filesDone, + } + } finally { + await unlink(tempPath).catch(() => undefined) } }