Add prepared tenant export archives
This commit is contained in:
@@ -1,5 +1,8 @@
|
||||
import { FastifyInstance } from "fastify"
|
||||
import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
|
||||
import archiver from "archiver"
|
||||
import { PassThrough } from "stream"
|
||||
import { BlobReader, TextWriter, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js"
|
||||
|
||||
import { pool } from "../../db"
|
||||
import { s3 } from "./s3"
|
||||
@@ -41,6 +44,34 @@ type ImportOptions = {
|
||||
targetTenantId?: number | null
|
||||
}
|
||||
|
||||
type BuildTenantFullExportOptions = {
|
||||
includeFileContents?: boolean
|
||||
}
|
||||
|
||||
type TenantArchiveManifest = {
|
||||
format: "fedeo.tenant-archive-export"
|
||||
version: 1
|
||||
exportedAt: string
|
||||
tenantId: number
|
||||
tables: { name: string; rows: number; path: string }[]
|
||||
files: {
|
||||
id: string
|
||||
path: string
|
||||
archivePath: string | null
|
||||
name: string | null
|
||||
mimeType: string | null
|
||||
size: number | null
|
||||
missing?: boolean
|
||||
error?: string
|
||||
}[]
|
||||
}
|
||||
|
||||
type ArchiveExportOptions = {
|
||||
filename?: string
|
||||
storagePath?: string
|
||||
onProgress?: (progress: { filesDone: number; filesTotal: number }) => Promise<void> | void
|
||||
}
|
||||
|
||||
const ENTITY_BANKACCOUNT_PLAIN_FIELDS = {
|
||||
iban: "__plainIban",
|
||||
bic: "__plainBic",
|
||||
@@ -196,8 +227,49 @@ const loadObjectAsBase64 = async (path: string) => {
|
||||
}
|
||||
}
|
||||
|
||||
export const buildTenantFullExport = async (server: FastifyInstance, tenantId: number): Promise<TenantFullExport> => {
|
||||
const loadObjectStream = async (path: string) => {
|
||||
try {
|
||||
const { Body } = await s3.send(new GetObjectCommand({
|
||||
Bucket: secrets.S3_BUCKET,
|
||||
Key: path,
|
||||
}))
|
||||
|
||||
return {
|
||||
body: Body as any,
|
||||
missing: false,
|
||||
error: null,
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (!isMissingObjectError(err)) throw err
|
||||
|
||||
return {
|
||||
body: null,
|
||||
missing: true,
|
||||
error: err?.Code || err?.name || "NoSuchKey",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const archiveEntryPathForFile = (path: string) => {
|
||||
const normalizedPath = path
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/^\/+/, "")
|
||||
.split("/")
|
||||
.filter((part) => part && part !== "." && part !== "..")
|
||||
.join("/")
|
||||
|
||||
return `files/${normalizedPath || "unnamed-file"}`
|
||||
}
|
||||
|
||||
const jsonBuffer = (value: unknown) => Buffer.from(`${JSON.stringify(value, null, 2)}\n`, "utf8")
|
||||
|
||||
export const buildTenantFullExport = async (
|
||||
server: FastifyInstance,
|
||||
tenantId: number,
|
||||
options: BuildTenantFullExportOptions = {}
|
||||
): Promise<TenantFullExport> => {
|
||||
const client = await pool.connect()
|
||||
const includeFileContents = options.includeFileContents !== false
|
||||
|
||||
try {
|
||||
const columnsByTable = await tableColumns(client)
|
||||
@@ -259,7 +331,9 @@ export const buildTenantFullExport = async (server: FastifyInstance, tenantId: n
|
||||
for (const file of fileRows) {
|
||||
if (!file.path) continue
|
||||
|
||||
const object = await loadObjectAsBase64(file.path)
|
||||
const object = includeFileContents
|
||||
? await loadObjectAsBase64(file.path)
|
||||
: { contentBase64: null, missing: false, error: null }
|
||||
if (object.missing) {
|
||||
server.log.warn({
|
||||
fileId: file.id,
|
||||
@@ -294,6 +368,106 @@ export const buildTenantFullExport = async (server: FastifyInstance, tenantId: n
|
||||
}
|
||||
}
|
||||
|
||||
export const createTenantFullExportArchive = async (
|
||||
server: FastifyInstance,
|
||||
tenantId: number,
|
||||
options: ArchiveExportOptions = {}
|
||||
) => {
|
||||
const exportData = await buildTenantFullExport(server, tenantId, { includeFileContents: false })
|
||||
const safeTenantName = String(exportData.tables.tenants?.[0]?.short || exportData.tables.tenants?.[0]?.name || tenantId)
|
||||
.replace(/[^a-z0-9_-]+/gi, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.toLowerCase()
|
||||
const filename = options.filename || `fedeo-tenant-${safeTenantName || tenantId}-${new Date().toISOString().slice(0, 10)}.fedeo-export.zip`
|
||||
const storagePath = options.storagePath || `tenant-exports/${tenantId}/${Date.now()}-${filename}`
|
||||
const fileRows = exportData.tables.files || []
|
||||
const archive = archiver("zip", { zlib: { level: 6 } })
|
||||
const output = new PassThrough()
|
||||
let size = 0
|
||||
|
||||
output.on("data", (chunk: Buffer) => {
|
||||
size += chunk.length
|
||||
})
|
||||
|
||||
const uploadPromise = s3.send(new PutObjectCommand({
|
||||
Bucket: secrets.S3_BUCKET,
|
||||
Key: storagePath,
|
||||
Body: output,
|
||||
ContentType: "application/zip",
|
||||
ContentDisposition: `attachment; filename="${filename}"`,
|
||||
}))
|
||||
|
||||
archive.pipe(output)
|
||||
|
||||
const tables = Object.entries(exportData.tables)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([table, rows]) => {
|
||||
const path = `tables/${table}.json`
|
||||
archive.append(jsonBuffer(rows), { name: path })
|
||||
return { name: table, rows: rows.length, path }
|
||||
})
|
||||
|
||||
const manifestFiles: TenantArchiveManifest["files"] = []
|
||||
let filesDone = 0
|
||||
const filesTotal = fileRows.filter((file) => file.path).length
|
||||
|
||||
for (const file of fileRows) {
|
||||
if (!file.path) continue
|
||||
|
||||
const archivePath = archiveEntryPathForFile(file.path)
|
||||
const object = await loadObjectStream(file.path)
|
||||
|
||||
if (object.missing) {
|
||||
server.log.warn({
|
||||
fileId: file.id,
|
||||
path: file.path,
|
||||
bucket: secrets.S3_BUCKET,
|
||||
error: object.error,
|
||||
}, "Tenant archive export skipped missing S3 object")
|
||||
} else {
|
||||
archive.append(object.body, { name: archivePath })
|
||||
}
|
||||
|
||||
filesDone += 1
|
||||
manifestFiles.push({
|
||||
id: file.id,
|
||||
path: file.path,
|
||||
archivePath: object.missing ? null : archivePath,
|
||||
name: file.name || null,
|
||||
mimeType: file.mimeType || null,
|
||||
size: file.size || null,
|
||||
missing: object.missing || undefined,
|
||||
error: object.error || undefined,
|
||||
})
|
||||
|
||||
if (filesDone % 10 === 0 || filesDone === filesTotal) {
|
||||
await options.onProgress?.({ filesDone, filesTotal })
|
||||
}
|
||||
}
|
||||
|
||||
const manifest: TenantArchiveManifest = {
|
||||
format: "fedeo.tenant-archive-export",
|
||||
version: 1,
|
||||
exportedAt: exportData.exportedAt,
|
||||
tenantId,
|
||||
tables,
|
||||
files: manifestFiles,
|
||||
}
|
||||
|
||||
archive.append(jsonBuffer(manifest), { name: "manifest.json" })
|
||||
await archive.finalize()
|
||||
await uploadPromise
|
||||
|
||||
return {
|
||||
filename,
|
||||
storagePath,
|
||||
contentType: "application/zip",
|
||||
size,
|
||||
filesTotal,
|
||||
filesDone,
|
||||
}
|
||||
}
|
||||
|
||||
const restoreFiles = async (exportData: TenantFullExport) => {
|
||||
let restored = 0
|
||||
let skipped = 0
|
||||
@@ -318,6 +492,56 @@ const restoreFiles = async (exportData: TenantFullExport) => {
|
||||
return { restored, skipped }
|
||||
}
|
||||
|
||||
const readZipTextEntry = async (entriesByName: Map<string, any>, name: string) => {
|
||||
const entry = entriesByName.get(name)
|
||||
if (!entry) throw new Error(`${name} fehlt im Mandantenexport`)
|
||||
|
||||
return entry.getData(new TextWriter())
|
||||
}
|
||||
|
||||
const restoreArchiveFiles = async (
|
||||
entriesByName: Map<string, any>,
|
||||
exportData: TenantFullExport,
|
||||
manifest: TenantArchiveManifest
|
||||
) => {
|
||||
let restored = 0
|
||||
let skipped = 0
|
||||
const filesByPath = new Map((manifest.files || []).map((file) => [file.path, file]))
|
||||
|
||||
for (const fileRow of exportData.tables.files || []) {
|
||||
const originalPath = fileRow.path
|
||||
if (!originalPath) {
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
|
||||
const manifestFile = filesByPath.get(originalPath)
|
||||
const archivePath = manifestFile?.archivePath
|
||||
if (!archivePath || manifestFile?.missing) {
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
|
||||
const entry = entriesByName.get(archivePath)
|
||||
if (!entry) {
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
|
||||
const content = await entry.getData(new Uint8ArrayWriter())
|
||||
await s3.send(new PutObjectCommand({
|
||||
Bucket: secrets.S3_BUCKET,
|
||||
Key: fileRow.path,
|
||||
Body: Buffer.from(content),
|
||||
ContentType: fileRow.mimeType || manifestFile.mimeType || "application/octet-stream",
|
||||
ContentLength: content.length,
|
||||
}))
|
||||
restored += 1
|
||||
}
|
||||
|
||||
return { restored, skipped }
|
||||
}
|
||||
|
||||
const remapTenantScopedExport = (
|
||||
exportData: TenantFullExport,
|
||||
targetTenantId?: number | null
|
||||
@@ -644,3 +868,70 @@ export const importTenantFullExport = async (
|
||||
client.release()
|
||||
}
|
||||
}
|
||||
|
||||
export const importTenantFullExportArchive = async (
|
||||
server: FastifyInstance,
|
||||
archiveBuffer: Buffer,
|
||||
options: ImportOptions = {}
|
||||
): Promise<ImportResult> => {
|
||||
const reader = new ZipReader(new BlobReader(new Blob([archiveBuffer])))
|
||||
|
||||
try {
|
||||
const entries = await reader.getEntries()
|
||||
const entriesByName = new Map(entries.map((entry: any) => [entry.filename, entry]))
|
||||
const manifest = JSON.parse(await readZipTextEntry(entriesByName, "manifest.json")) as TenantArchiveManifest
|
||||
|
||||
if (manifest?.format !== "fedeo.tenant-archive-export" || manifest.version !== 1) {
|
||||
throw new Error("Ungültiges FEDEO Mandantenarchiv-Format")
|
||||
}
|
||||
|
||||
const tables: TableRows = {}
|
||||
for (const table of manifest.tables || []) {
|
||||
tables[table.name] = JSON.parse(await readZipTextEntry(entriesByName, table.path))
|
||||
}
|
||||
|
||||
const rawExportData: TenantFullExport = {
|
||||
format: "fedeo.tenant-full-export",
|
||||
version: 1,
|
||||
exportedAt: manifest.exportedAt,
|
||||
tenantId: manifest.tenantId,
|
||||
tables,
|
||||
files: (manifest.files || []).map((file) => ({
|
||||
id: file.id,
|
||||
path: file.path,
|
||||
name: file.name,
|
||||
mimeType: file.mimeType,
|
||||
size: file.size,
|
||||
contentBase64: null,
|
||||
missing: file.missing,
|
||||
error: file.error,
|
||||
})),
|
||||
}
|
||||
|
||||
const exportData = remapTenantScopedExport(rawExportData, options.targetTenantId)
|
||||
const sourceTenantId = rawExportData.tenantId
|
||||
const targetTenantId = exportData.tenantId
|
||||
const sourcePrefix = `${sourceTenantId}/`
|
||||
const targetPrefix = `${targetTenantId}/`
|
||||
const remappedManifest: TenantArchiveManifest = {
|
||||
...manifest,
|
||||
tenantId: targetTenantId,
|
||||
files: (manifest.files || []).map((file) => ({
|
||||
...file,
|
||||
path: file.path?.startsWith(sourcePrefix)
|
||||
? `${targetPrefix}${file.path.slice(sourcePrefix.length)}`
|
||||
: file.path,
|
||||
})),
|
||||
}
|
||||
|
||||
const result = await importTenantFullExport(server, exportData, { targetTenantId: null })
|
||||
const files = await restoreArchiveFiles(entriesByName, exportData, remappedManifest)
|
||||
|
||||
return {
|
||||
...result,
|
||||
files,
|
||||
}
|
||||
} finally {
|
||||
await reader.close()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user