992 lines
34 KiB
TypeScript
992 lines
34 KiB
TypeScript
import { FastifyInstance } from "fastify"
|
|
import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
|
|
import archiver from "archiver"
|
|
import { createReadStream, createWriteStream } from "fs"
|
|
import { stat, unlink } from "fs/promises"
|
|
import os from "os"
|
|
import path from "path"
|
|
import { BlobReader, TextWriter, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js"
|
|
|
|
import { pool } from "../../db"
|
|
import { s3 } from "./s3"
|
|
import { secrets } from "./secrets"
|
|
import { decrypt, encrypt } from "./crypt"
|
|
|
|
type TableRows = Record<string, Record<string, any>[]>
|
|
type TableMetadata = {
|
|
columns: string[]
|
|
jsonColumns: Set<string>
|
|
generatedColumns: Set<string>
|
|
}
|
|
|
|
export type TenantFullExport = {
|
|
format: "fedeo.tenant-full-export"
|
|
version: 1
|
|
exportedAt: string
|
|
tenantId: number
|
|
tables: TableRows
|
|
files: {
|
|
id: string
|
|
path: string
|
|
name: string | null
|
|
mimeType: string | null
|
|
size: number | null
|
|
contentBase64: string | null
|
|
missing?: boolean
|
|
error?: string
|
|
}[]
|
|
}
|
|
|
|
type ImportResult = {
|
|
tenantId: number
|
|
tables: { table: string; rows: number }[]
|
|
files: { restored: number; skipped: number }
|
|
}
|
|
|
|
type ImportOptions = {
|
|
targetTenantId?: number | null
|
|
onProgress?: (progress: { done: number; total: number; message?: string }) => Promise<void> | void
|
|
}
|
|
|
|
type BuildTenantFullExportOptions = {
|
|
includeFileContents?: boolean
|
|
}
|
|
|
|
type TenantArchiveManifest = {
|
|
format: "fedeo.tenant-archive-export"
|
|
version: 1
|
|
exportedAt: string
|
|
tenantId: number
|
|
tables: { name: string; rows: number; path: string }[]
|
|
files: {
|
|
id: string
|
|
path: string
|
|
archivePath: string | null
|
|
name: string | null
|
|
mimeType: string | null
|
|
size: number | null
|
|
missing?: boolean
|
|
error?: string
|
|
}[]
|
|
}
|
|
|
|
type ArchiveExportOptions = {
|
|
filename?: string
|
|
storagePath?: string
|
|
onProgress?: (progress: { filesDone: number; filesTotal: number }) => Promise<void> | void
|
|
}
|
|
|
|
const ENTITY_BANKACCOUNT_PLAIN_FIELDS = {
|
|
iban: "__plainIban",
|
|
bic: "__plainBic",
|
|
bankName: "__plainBankName",
|
|
}
|
|
|
|
const quoteIdent = (value: string) => `"${value.replace(/"/g, '""')}"`
|
|
const matrixServerName = () =>
|
|
process.env.MATRIX_SERVER_NAME ||
|
|
secrets.MATRIX_SERVER_NAME ||
|
|
process.env.DOMAIN ||
|
|
"localhost"
|
|
|
|
const normalizeMatrixLocalpartSeed = (value: string) => {
|
|
const normalized = value
|
|
.toLowerCase()
|
|
.normalize("NFKD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/ä/g, "a")
|
|
.replace(/ö/g, "o")
|
|
.replace(/ü/g, "u")
|
|
.replace(/ß/g, "ss")
|
|
.replace(/[^a-z0-9._=-]+/g, "_")
|
|
.replace(/_+/g, "_")
|
|
.replace(/^[._=-]+|[._=-]+$/g, "")
|
|
|
|
return normalized || "user"
|
|
}
|
|
|
|
const normalizeMatrixAliasSeed = (value: string) =>
|
|
normalizeMatrixLocalpartSeed(value)
|
|
.replace(/[.=]/g, "_")
|
|
.replace(/_+/g, "_")
|
|
|
|
const tenantRoomAliasLocalpart = (
|
|
tenant: { id: number, short?: string | null, name?: string | null },
|
|
roomKey: string
|
|
) => {
|
|
const tenantSeed = normalizeMatrixAliasSeed(tenant.short || tenant.name || `tenant_${tenant.id}`)
|
|
const roomSeed = normalizeMatrixAliasSeed(roomKey)
|
|
return `fedeo_${tenantSeed}_${tenant.id}_${roomSeed}`
|
|
}
|
|
|
|
const tenantRoomAlias = (
|
|
tenant: { id: number, short?: string | null, name?: string | null },
|
|
roomKey: string
|
|
) => `#${tenantRoomAliasLocalpart(tenant, roomKey)}:${matrixServerName()}`
|
|
|
|
const tableColumns = async (client: any) => {
|
|
const result = await client.query(`
|
|
select table_name, column_name, data_type, is_generated
|
|
from information_schema.columns
|
|
where table_schema = 'public'
|
|
order by table_name, ordinal_position
|
|
`)
|
|
|
|
const columnsByTable = new Map<string, TableMetadata>()
|
|
for (const row of result.rows) {
|
|
const metadata = columnsByTable.get(row.table_name) || {
|
|
columns: [],
|
|
jsonColumns: new Set<string>(),
|
|
generatedColumns: new Set<string>(),
|
|
}
|
|
|
|
metadata.columns.push(row.column_name)
|
|
if (row.data_type === "json" || row.data_type === "jsonb") {
|
|
metadata.jsonColumns.add(row.column_name)
|
|
}
|
|
if (row.is_generated === "ALWAYS") {
|
|
metadata.generatedColumns.add(row.column_name)
|
|
}
|
|
|
|
columnsByTable.set(row.table_name, metadata)
|
|
}
|
|
|
|
return columnsByTable
|
|
}
|
|
|
|
const loadRows = async (client: any, table: string, whereSql: string, params: any[] = []) => {
|
|
const result = await client.query(`select * from ${quoteIdent(table)} where ${whereSql}`, params)
|
|
|
|
return result.rows
|
|
}
|
|
|
|
const collectIds = (rows: Record<string, any>[], column: string) => {
|
|
return Array.from(new Set(rows.map((row) => row[column]).filter(Boolean)))
|
|
}
|
|
|
|
const addRows = (tables: TableRows, table: string, rows: Record<string, any>[]) => {
|
|
if (!rows.length) {
|
|
if (!tables[table]) tables[table] = []
|
|
return
|
|
}
|
|
|
|
const existingRows = tables[table] || []
|
|
const existingKeys = new Set(existingRows.map((row) => JSON.stringify(row)))
|
|
for (const row of rows) {
|
|
const key = JSON.stringify(row)
|
|
if (!existingKeys.has(key)) {
|
|
existingRows.push(row)
|
|
existingKeys.add(key)
|
|
}
|
|
}
|
|
tables[table] = existingRows
|
|
}
|
|
|
|
const decryptEntityBankAccountsForExport = (rows: Record<string, any>[] = []) => {
|
|
return rows.map((row) => {
|
|
const nextRow = { ...row }
|
|
|
|
try {
|
|
nextRow[ENTITY_BANKACCOUNT_PLAIN_FIELDS.iban] = row.iban_encrypted ? decrypt(row.iban_encrypted) : null
|
|
nextRow[ENTITY_BANKACCOUNT_PLAIN_FIELDS.bic] = row.bic_encrypted ? decrypt(row.bic_encrypted) : null
|
|
nextRow[ENTITY_BANKACCOUNT_PLAIN_FIELDS.bankName] = row.bank_name_encrypted ? decrypt(row.bank_name_encrypted) : null
|
|
} catch (err: any) {
|
|
throw new Error(`Bankverbindung ${row.id || ""} konnte für den Export nicht entschlüsselt werden: ${err?.message || err}`)
|
|
}
|
|
|
|
return nextRow
|
|
})
|
|
}
|
|
|
|
const isMissingObjectError = (err: any) =>
|
|
err?.Code === "NoSuchKey" ||
|
|
err?.name === "NoSuchKey" ||
|
|
err?.$metadata?.httpStatusCode === 404
|
|
|
|
const loadObjectAsBase64 = async (path: string) => {
|
|
try {
|
|
const { Body } = await s3.send(new GetObjectCommand({
|
|
Bucket: secrets.S3_BUCKET,
|
|
Key: path,
|
|
}))
|
|
|
|
const chunks: Buffer[] = []
|
|
for await (const chunk of Body as any) {
|
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
|
|
}
|
|
|
|
return {
|
|
contentBase64: Buffer.concat(chunks).toString("base64"),
|
|
missing: false,
|
|
error: null,
|
|
}
|
|
} catch (err: any) {
|
|
if (!isMissingObjectError(err)) throw err
|
|
|
|
return {
|
|
contentBase64: null,
|
|
missing: true,
|
|
error: err?.Code || err?.name || "NoSuchKey",
|
|
}
|
|
}
|
|
}
|
|
|
|
const loadObjectStream = async (path: string) => {
|
|
try {
|
|
const { Body } = await s3.send(new GetObjectCommand({
|
|
Bucket: secrets.S3_BUCKET,
|
|
Key: path,
|
|
}))
|
|
|
|
return {
|
|
body: Body as any,
|
|
missing: false,
|
|
error: null,
|
|
}
|
|
} catch (err: any) {
|
|
if (!isMissingObjectError(err)) throw err
|
|
|
|
return {
|
|
body: null,
|
|
missing: true,
|
|
error: err?.Code || err?.name || "NoSuchKey",
|
|
}
|
|
}
|
|
}
|
|
|
|
const archiveEntryPathForFile = (path: string) => {
|
|
const normalizedPath = path
|
|
.replace(/\\/g, "/")
|
|
.replace(/^\/+/, "")
|
|
.split("/")
|
|
.filter((part) => part && part !== "." && part !== "..")
|
|
.join("/")
|
|
|
|
return `files/${normalizedPath || "unnamed-file"}`
|
|
}
|
|
|
|
const jsonBuffer = (value: unknown) => Buffer.from(`${JSON.stringify(value, null, 2)}\n`, "utf8")
|
|
|
|
export const buildTenantFullExport = async (
|
|
server: FastifyInstance,
|
|
tenantId: number,
|
|
options: BuildTenantFullExportOptions = {}
|
|
): Promise<TenantFullExport> => {
|
|
const client = await pool.connect()
|
|
const includeFileContents = options.includeFileContents !== false
|
|
|
|
try {
|
|
const columnsByTable = await tableColumns(client)
|
|
const tables: TableRows = {}
|
|
|
|
const tenantRows = await loadRows(client, "tenants", "id = $1", [tenantId])
|
|
if (!tenantRows.length) throw new Error("Tenant nicht gefunden")
|
|
|
|
addRows(tables, "tenants", tenantRows)
|
|
|
|
for (const [table, metadata] of columnsByTable.entries()) {
|
|
if (table === "tenants") continue
|
|
const { columns } = metadata
|
|
|
|
const tenantColumn = columns.includes("tenant")
|
|
? "tenant"
|
|
: columns.includes("tenant_id")
|
|
? "tenant_id"
|
|
: null
|
|
|
|
if (!tenantColumn) continue
|
|
|
|
const rows = await loadRows(client, table, `${quoteIdent(tenantColumn)} = $1`, [tenantId])
|
|
addRows(tables, table, rows)
|
|
}
|
|
|
|
const profileIds = collectIds(tables.auth_profiles || [], "id")
|
|
const userIds = Array.from(new Set([
|
|
...collectIds(tables.auth_tenant_users || [], "user_id"),
|
|
...collectIds(tables.auth_profiles || [], "user_id"),
|
|
...collectIds(tables.auth_user_roles || [], "user_id"),
|
|
]))
|
|
const roleIds = Array.from(new Set([
|
|
...collectIds(tables.auth_roles || [], "id"),
|
|
...collectIds(tables.auth_user_roles || [], "role_id"),
|
|
]))
|
|
|
|
if (userIds.length) {
|
|
addRows(tables, "auth_users", await loadRows(client, "auth_users", "id = any($1::uuid[])", [userIds]))
|
|
}
|
|
|
|
if (roleIds.length) {
|
|
addRows(tables, "auth_roles", await loadRows(client, "auth_roles", "id = any($1::uuid[])", [roleIds]))
|
|
addRows(tables, "auth_role_permissions", await loadRows(client, "auth_role_permissions", "role_id = any($1::uuid[])", [roleIds]))
|
|
}
|
|
|
|
if (profileIds.length) {
|
|
addRows(tables, "auth_profile_branches", await loadRows(client, "auth_profile_branches", "profile_id = any($1::uuid[])", [profileIds]))
|
|
addRows(tables, "auth_profile_teams", await loadRows(client, "auth_profile_teams", "profile_id = any($1::uuid[])", [profileIds]))
|
|
}
|
|
|
|
if (tables.entitybankaccounts?.length) {
|
|
tables.entitybankaccounts = decryptEntityBankAccountsForExport(tables.entitybankaccounts)
|
|
}
|
|
|
|
const fileRows = tables.files || []
|
|
const files = []
|
|
|
|
for (const file of fileRows) {
|
|
if (!file.path) continue
|
|
|
|
const object = includeFileContents
|
|
? await loadObjectAsBase64(file.path)
|
|
: { contentBase64: null, missing: false, error: null }
|
|
if (object.missing) {
|
|
server.log.warn({
|
|
fileId: file.id,
|
|
path: file.path,
|
|
bucket: secrets.S3_BUCKET,
|
|
error: object.error,
|
|
}, "Tenant full export skipped missing S3 object")
|
|
}
|
|
|
|
files.push({
|
|
id: file.id,
|
|
path: file.path,
|
|
name: file.name || null,
|
|
mimeType: file.mimeType || null,
|
|
size: file.size || null,
|
|
contentBase64: object.contentBase64,
|
|
missing: object.missing || undefined,
|
|
error: object.error || undefined,
|
|
})
|
|
}
|
|
|
|
return {
|
|
format: "fedeo.tenant-full-export",
|
|
version: 1,
|
|
exportedAt: new Date().toISOString(),
|
|
tenantId,
|
|
tables,
|
|
files,
|
|
}
|
|
} finally {
|
|
client.release()
|
|
}
|
|
}
|
|
|
|
export const createTenantFullExportArchive = async (
|
|
server: FastifyInstance,
|
|
tenantId: number,
|
|
options: ArchiveExportOptions = {}
|
|
) => {
|
|
const exportData = await buildTenantFullExport(server, tenantId, { includeFileContents: false })
|
|
const safeTenantName = String(exportData.tables.tenants?.[0]?.short || exportData.tables.tenants?.[0]?.name || tenantId)
|
|
.replace(/[^a-z0-9_-]+/gi, "-")
|
|
.replace(/^-+|-+$/g, "")
|
|
.toLowerCase()
|
|
const filename = options.filename || `fedeo-tenant-${safeTenantName || tenantId}-${new Date().toISOString().slice(0, 10)}.fedeo-export.zip`
|
|
const storagePath = options.storagePath || `tenant-exports/${tenantId}/${Date.now()}-${filename}`
|
|
const fileRows = exportData.tables.files || []
|
|
const archive = archiver("zip", { zlib: { level: 6 } })
|
|
const tempPath = path.join(os.tmpdir(), `fedeo-tenant-export-${tenantId}-${Date.now()}-${filename}`)
|
|
const output = createWriteStream(tempPath)
|
|
const archiveComplete = new Promise<void>((resolve, reject) => {
|
|
output.on("close", resolve)
|
|
output.on("error", reject)
|
|
archive.on("error", reject)
|
|
archive.on("warning", (err) => {
|
|
if ((err as NodeJS.ErrnoException).code === "ENOENT") return
|
|
reject(err)
|
|
})
|
|
})
|
|
|
|
archive.pipe(output)
|
|
|
|
try {
|
|
const tables = Object.entries(exportData.tables)
|
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
.map(([table, rows]) => {
|
|
const path = `tables/${table}.json`
|
|
archive.append(jsonBuffer(rows), { name: path })
|
|
return { name: table, rows: rows.length, path }
|
|
})
|
|
|
|
const manifestFiles: TenantArchiveManifest["files"] = []
|
|
let filesDone = 0
|
|
const filesTotal = fileRows.filter((file) => file.path).length
|
|
|
|
for (const file of fileRows) {
|
|
if (!file.path) continue
|
|
|
|
const archivePath = archiveEntryPathForFile(file.path)
|
|
const object = await loadObjectStream(file.path)
|
|
|
|
if (object.missing) {
|
|
server.log.warn({
|
|
fileId: file.id,
|
|
path: file.path,
|
|
bucket: secrets.S3_BUCKET,
|
|
error: object.error,
|
|
}, "Tenant archive export skipped missing S3 object")
|
|
} else {
|
|
archive.append(object.body, { name: archivePath })
|
|
}
|
|
|
|
filesDone += 1
|
|
manifestFiles.push({
|
|
id: file.id,
|
|
path: file.path,
|
|
archivePath: object.missing ? null : archivePath,
|
|
name: file.name || null,
|
|
mimeType: file.mimeType || null,
|
|
size: file.size || null,
|
|
missing: object.missing || undefined,
|
|
error: object.error || undefined,
|
|
})
|
|
|
|
if (filesDone % 10 === 0 || filesDone === filesTotal) {
|
|
await options.onProgress?.({ filesDone, filesTotal })
|
|
}
|
|
}
|
|
|
|
const manifest: TenantArchiveManifest = {
|
|
format: "fedeo.tenant-archive-export",
|
|
version: 1,
|
|
exportedAt: exportData.exportedAt,
|
|
tenantId,
|
|
tables,
|
|
files: manifestFiles,
|
|
}
|
|
|
|
archive.append(jsonBuffer(manifest), { name: "manifest.json" })
|
|
await archive.finalize()
|
|
await archiveComplete
|
|
|
|
const { size } = await stat(tempPath)
|
|
await s3.send(new PutObjectCommand({
|
|
Bucket: secrets.S3_BUCKET,
|
|
Key: storagePath,
|
|
Body: createReadStream(tempPath),
|
|
ContentLength: size,
|
|
ContentType: "application/zip",
|
|
ContentDisposition: `attachment; filename="${filename}"`,
|
|
}))
|
|
|
|
return {
|
|
filename,
|
|
storagePath,
|
|
contentType: "application/zip",
|
|
size,
|
|
filesTotal,
|
|
filesDone,
|
|
}
|
|
} finally {
|
|
await unlink(tempPath).catch(() => undefined)
|
|
}
|
|
}
|
|
|
|
const restoreFiles = async (exportData: TenantFullExport) => {
|
|
let restored = 0
|
|
let skipped = 0
|
|
|
|
for (const file of exportData.files || []) {
|
|
if (!file.path || !file.contentBase64) {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
|
|
const body = Buffer.from(file.contentBase64, "base64")
|
|
await s3.send(new PutObjectCommand({
|
|
Bucket: secrets.S3_BUCKET,
|
|
Key: file.path,
|
|
Body: body,
|
|
ContentType: file.mimeType || "application/octet-stream",
|
|
ContentLength: body.length,
|
|
}))
|
|
restored += 1
|
|
}
|
|
|
|
return { restored, skipped }
|
|
}
|
|
|
|
const readZipTextEntry = async (entriesByName: Map<string, any>, name: string) => {
|
|
const entry = entriesByName.get(name)
|
|
if (!entry) throw new Error(`${name} fehlt im Mandantenexport`)
|
|
|
|
return entry.getData(new TextWriter())
|
|
}
|
|
|
|
const restoreArchiveFiles = async (
|
|
entriesByName: Map<string, any>,
|
|
exportData: TenantFullExport,
|
|
manifest: TenantArchiveManifest
|
|
) => {
|
|
let restored = 0
|
|
let skipped = 0
|
|
const filesByPath = new Map((manifest.files || []).map((file) => [file.path, file]))
|
|
|
|
for (const fileRow of exportData.tables.files || []) {
|
|
const originalPath = fileRow.path
|
|
if (!originalPath) {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
|
|
const manifestFile = filesByPath.get(originalPath)
|
|
const archivePath = manifestFile?.archivePath
|
|
if (!archivePath || manifestFile?.missing) {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
|
|
const entry = entriesByName.get(archivePath)
|
|
if (!entry) {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
|
|
const content = await entry.getData(new Uint8ArrayWriter())
|
|
await s3.send(new PutObjectCommand({
|
|
Bucket: secrets.S3_BUCKET,
|
|
Key: fileRow.path,
|
|
Body: Buffer.from(content),
|
|
ContentType: fileRow.mimeType || manifestFile.mimeType || "application/octet-stream",
|
|
ContentLength: content.length,
|
|
}))
|
|
restored += 1
|
|
}
|
|
|
|
return { restored, skipped }
|
|
}
|
|
|
|
const remapTenantScopedExport = (
|
|
exportData: TenantFullExport,
|
|
targetTenantId?: number | null
|
|
): TenantFullExport => {
|
|
if (!targetTenantId || targetTenantId === exportData.tenantId) return exportData
|
|
|
|
const sourceTenantId = exportData.tenantId
|
|
const sourcePathPrefix = `${sourceTenantId}/`
|
|
const targetPathPrefix = `${targetTenantId}/`
|
|
const tables: TableRows = {}
|
|
|
|
for (const [table, rows] of Object.entries(exportData.tables || {})) {
|
|
tables[table] = rows.map((row) => {
|
|
const nextRow = { ...row }
|
|
|
|
if (table === "tenants" && nextRow.id === sourceTenantId) {
|
|
nextRow.id = targetTenantId
|
|
}
|
|
|
|
if (nextRow.tenant === sourceTenantId) {
|
|
nextRow.tenant = targetTenantId
|
|
}
|
|
|
|
if (nextRow.tenant_id === sourceTenantId) {
|
|
nextRow.tenant_id = targetTenantId
|
|
}
|
|
|
|
if (table === "files" && typeof nextRow.path === "string" && nextRow.path.startsWith(sourcePathPrefix)) {
|
|
nextRow.path = `${targetPathPrefix}${nextRow.path.slice(sourcePathPrefix.length)}`
|
|
}
|
|
|
|
if (table === "letterheads" && typeof nextRow.path === "string" && nextRow.path.startsWith(sourcePathPrefix)) {
|
|
nextRow.path = `${targetPathPrefix}${nextRow.path.slice(sourcePathPrefix.length)}`
|
|
}
|
|
|
|
return nextRow
|
|
})
|
|
}
|
|
|
|
return {
|
|
...exportData,
|
|
tenantId: targetTenantId,
|
|
tables,
|
|
files: (exportData.files || []).map((file) => ({
|
|
...file,
|
|
path: file.path?.startsWith(sourcePathPrefix)
|
|
? `${targetPathPrefix}${file.path.slice(sourcePathPrefix.length)}`
|
|
: file.path,
|
|
})),
|
|
}
|
|
}
|
|
|
|
const encryptEntityBankAccountRowsForImport = (exportData: TenantFullExport) => {
|
|
const rows = exportData.tables.entitybankaccounts || []
|
|
|
|
for (const row of rows) {
|
|
const plainIban = row[ENTITY_BANKACCOUNT_PLAIN_FIELDS.iban]
|
|
const plainBic = row[ENTITY_BANKACCOUNT_PLAIN_FIELDS.bic]
|
|
const plainBankName = row[ENTITY_BANKACCOUNT_PLAIN_FIELDS.bankName]
|
|
|
|
if (typeof plainIban === "string" && typeof plainBic === "string" && typeof plainBankName === "string") {
|
|
row.iban_encrypted = encrypt(plainIban)
|
|
row.bic_encrypted = encrypt(plainBic)
|
|
row.bank_name_encrypted = encrypt(plainBankName)
|
|
}
|
|
|
|
delete row[ENTITY_BANKACCOUNT_PLAIN_FIELDS.iban]
|
|
delete row[ENTITY_BANKACCOUNT_PLAIN_FIELDS.bic]
|
|
delete row[ENTITY_BANKACCOUNT_PLAIN_FIELDS.bankName]
|
|
}
|
|
}
|
|
|
|
const prepareCommunicationRoomsForImport = (exportData: TenantFullExport) => {
|
|
const rows = exportData.tables.communication_rooms || []
|
|
if (!rows.length) return
|
|
|
|
const tenantById = new Map((exportData.tables.tenants || []).map((tenant) => [
|
|
Number(tenant.id),
|
|
{
|
|
id: Number(tenant.id),
|
|
name: tenant.name,
|
|
short: tenant.short,
|
|
},
|
|
]))
|
|
|
|
for (const row of rows) {
|
|
const tenantId = Number(row.tenant_id)
|
|
const tenant = tenantById.get(tenantId)
|
|
|
|
row.matrix_room_id = null
|
|
row.parent_space_room_id = null
|
|
|
|
if (tenant && row.key) {
|
|
row.matrix_alias = tenantRoomAlias(tenant, String(row.key))
|
|
} else {
|
|
row.matrix_alias = null
|
|
}
|
|
}
|
|
}
|
|
|
|
const cleanupImportedCommunicationRooms = async (client: any, exportData: TenantFullExport) => {
|
|
const rows = exportData.tables.communication_rooms || []
|
|
if (!rows.length) return 0
|
|
|
|
const tenantById = new Map((exportData.tables.tenants || []).map((tenant) => [
|
|
Number(tenant.id),
|
|
{
|
|
id: Number(tenant.id),
|
|
name: tenant.name,
|
|
short: tenant.short,
|
|
},
|
|
]))
|
|
let cleaned = 0
|
|
|
|
for (const row of rows) {
|
|
const tenantId = Number(row.tenant_id)
|
|
const key = String(row.key || "")
|
|
const tenant = tenantById.get(tenantId)
|
|
if (!tenantId || !key || !tenant) continue
|
|
|
|
const alias = tenantRoomAlias(tenant, key)
|
|
const result = await client.query(
|
|
`
|
|
update communication_rooms
|
|
set matrix_room_id = null,
|
|
parent_space_room_id = null,
|
|
matrix_alias = $3,
|
|
updated_at = now()
|
|
where tenant_id = $1 and key = $2
|
|
`,
|
|
[tenantId, key, alias]
|
|
)
|
|
|
|
cleaned += result.rowCount || 0
|
|
}
|
|
|
|
return cleaned
|
|
}
|
|
|
|
const prepareColumnValue = (value: any, isJsonColumn: boolean) => {
|
|
if (!isJsonColumn || value === null || typeof value === "undefined") return value
|
|
if (typeof value === "string") return value
|
|
|
|
return JSON.stringify(value)
|
|
}
|
|
|
|
const insertRows = async (client: any, table: string, rows: Record<string, any>[], metadata: TableMetadata) => {
|
|
if (!rows.length) return 0
|
|
|
|
let inserted = 0
|
|
const availableColumns = new Set(metadata.columns.filter((column) => !metadata.generatedColumns.has(column)))
|
|
|
|
for (const row of rows) {
|
|
const rowColumns = Object.keys(row).filter((column) => availableColumns.has(column))
|
|
if (!rowColumns.length) continue
|
|
|
|
const placeholders = rowColumns.map((_, index) => `$${index + 1}`).join(", ")
|
|
const values = rowColumns.map((column) => prepareColumnValue(row[column], metadata.jsonColumns.has(column)))
|
|
|
|
await client.query(
|
|
`insert into ${quoteIdent(table)} (${rowColumns.map(quoteIdent).join(", ")}) values (${placeholders}) on conflict do nothing`,
|
|
values
|
|
)
|
|
inserted += 1
|
|
}
|
|
|
|
return inserted
|
|
}
|
|
|
|
const insertIdentityRowsWithRemap = async (
|
|
client: any,
|
|
table: string,
|
|
rows: Record<string, any>[],
|
|
metadata: TableMetadata
|
|
) => {
|
|
if (!rows.length) return { count: 0, idMap: new Map<number, number>() }
|
|
|
|
let count = 0
|
|
const idMap = new Map<number, number>()
|
|
const availableColumns = new Set(metadata.columns.filter((column) => !metadata.generatedColumns.has(column)))
|
|
|
|
for (const row of rows) {
|
|
const sourceId = Number(row.id)
|
|
if (!sourceId) continue
|
|
|
|
const existing = await client.query(`select * from ${quoteIdent(table)} where ${quoteIdent("id")} = $1 limit 1`, [sourceId])
|
|
const existingTenant = existing.rows[0]?.tenant ?? existing.rows[0]?.tenant_id
|
|
const rowTenant = row.tenant ?? row.tenant_id
|
|
const shouldPreserveId = existing.rows.length === 0
|
|
const shouldReuseExisting = existing.rows.length > 0 && rowTenant && existingTenant === rowTenant
|
|
const rowForInsert = shouldPreserveId ? row : { ...row, id: undefined }
|
|
const rowColumns = Object
|
|
.keys(rowForInsert)
|
|
.filter((column) => availableColumns.has(column) && typeof rowForInsert[column] !== "undefined")
|
|
|
|
if (shouldReuseExisting) {
|
|
idMap.set(sourceId, sourceId)
|
|
continue
|
|
}
|
|
|
|
if (!rowColumns.length) continue
|
|
|
|
const placeholders = rowColumns.map((_, index) => `$${index + 1}`).join(", ")
|
|
const values = rowColumns.map((column) => prepareColumnValue(rowForInsert[column], metadata.jsonColumns.has(column)))
|
|
const inserted = await client.query(
|
|
`insert into ${quoteIdent(table)} (${rowColumns.map(quoteIdent).join(", ")}) values (${placeholders}) returning ${quoteIdent("id")}`,
|
|
values
|
|
)
|
|
const targetId = Number(inserted.rows[0]?.id)
|
|
if (targetId) {
|
|
idMap.set(sourceId, targetId)
|
|
}
|
|
count += 1
|
|
}
|
|
|
|
return { count, idMap }
|
|
}
|
|
|
|
const remapTableColumn = (rows: Record<string, any>[] = [], column: string, idMap: Map<number, number>) => {
|
|
if (!idMap.size) return
|
|
|
|
for (const row of rows) {
|
|
const currentValue = Number(row[column])
|
|
const mappedValue = idMap.get(currentValue)
|
|
if (mappedValue) row[column] = mappedValue
|
|
}
|
|
}
|
|
|
|
const refreshSequences = async (client: any, columnsByTable: Map<string, TableMetadata>) => {
|
|
for (const [table, metadata] of columnsByTable.entries()) {
|
|
const { columns } = metadata
|
|
if (!columns.includes("id")) continue
|
|
|
|
const sequenceResult = await client.query("select pg_get_serial_sequence($1, $2) as sequence_name", [`public.${table}`, "id"])
|
|
const sequenceName = sequenceResult.rows[0]?.sequence_name
|
|
if (!sequenceName) continue
|
|
|
|
await client.query(`
|
|
select setval(
|
|
$1::regclass,
|
|
greatest(coalesce((select max(id) from ${quoteIdent(table)}), 1), 1),
|
|
true
|
|
)
|
|
`, [sequenceName])
|
|
}
|
|
}
|
|
|
|
export const importTenantFullExport = async (
|
|
server: FastifyInstance,
|
|
rawExportData: TenantFullExport,
|
|
options: ImportOptions = {}
|
|
): Promise<ImportResult> => {
|
|
if (rawExportData?.format !== "fedeo.tenant-full-export" || rawExportData.version !== 1) {
|
|
throw new Error("Ungültiges FEDEO Mandantenexport-Format")
|
|
}
|
|
|
|
const exportData = remapTenantScopedExport(rawExportData, options.targetTenantId)
|
|
encryptEntityBankAccountRowsForImport(exportData)
|
|
prepareCommunicationRoomsForImport(exportData)
|
|
const client = await pool.connect()
|
|
const importOrder = [
|
|
"tenants",
|
|
"auth_users",
|
|
"auth_roles",
|
|
"auth_role_permissions",
|
|
"auth_tenant_users",
|
|
"auth_profiles",
|
|
"auth_user_roles",
|
|
"auth_profile_branches",
|
|
"auth_profile_teams",
|
|
]
|
|
|
|
try {
|
|
const columnsByTable = await tableColumns(client)
|
|
const tableNames = [
|
|
...importOrder,
|
|
...Object.keys(exportData.tables).filter((table) => !importOrder.includes(table)).sort(),
|
|
].filter((table, index, all) => all.indexOf(table) === index)
|
|
const specialTables = [
|
|
columnsByTable.has("bankaccounts") ? "bankaccounts" : null,
|
|
columnsByTable.has("bankstatements") ? "bankstatements" : null,
|
|
columnsByTable.has("letterheads") ? "letterheads" : null,
|
|
].filter(Boolean) as string[]
|
|
const regularTableNames = tableNames.filter((table) => !specialTables.includes(table))
|
|
const progressTotal = Math.max(1, specialTables.length + regularTableNames.length + 3)
|
|
let progressDone = 0
|
|
const reportProgress = async (message: string) => {
|
|
await options.onProgress?.({
|
|
done: Math.min(progressDone, progressTotal),
|
|
total: progressTotal,
|
|
message,
|
|
})
|
|
}
|
|
|
|
await client.query("begin")
|
|
await client.query("set local session_replication_role = replica")
|
|
await reportProgress("Dateien werden vorbereitet")
|
|
const files = await restoreFiles(exportData)
|
|
progressDone += 1
|
|
await reportProgress("Dateien vorbereitet")
|
|
|
|
const importedTables: { table: string; rows: number }[] = []
|
|
const bankaccountMetadata = columnsByTable.get("bankaccounts")
|
|
if (bankaccountMetadata) {
|
|
const result = await insertIdentityRowsWithRemap(client, "bankaccounts", exportData.tables.bankaccounts || [], bankaccountMetadata)
|
|
remapTableColumn(exportData.tables.bankstatements, "account", result.idMap)
|
|
importedTables.push({ table: "bankaccounts", rows: result.count })
|
|
progressDone += 1
|
|
await reportProgress("Bankkonten importiert")
|
|
}
|
|
|
|
const bankstatementMetadata = columnsByTable.get("bankstatements")
|
|
if (bankstatementMetadata) {
|
|
const result = await insertIdentityRowsWithRemap(client, "bankstatements", exportData.tables.bankstatements || [], bankstatementMetadata)
|
|
remapTableColumn(exportData.tables.statementallocations, "bs_id", result.idMap)
|
|
remapTableColumn(exportData.tables.historyitems, "bankstatement", result.idMap)
|
|
importedTables.push({ table: "bankstatements", rows: result.count })
|
|
progressDone += 1
|
|
await reportProgress("Bankauszüge importiert")
|
|
}
|
|
|
|
const letterheadMetadata = columnsByTable.get("letterheads")
|
|
if (letterheadMetadata) {
|
|
const result = await insertIdentityRowsWithRemap(client, "letterheads", exportData.tables.letterheads || [], letterheadMetadata)
|
|
remapTableColumn(exportData.tables.createddocuments, "letterhead", result.idMap)
|
|
importedTables.push({ table: "letterheads", rows: result.count })
|
|
progressDone += 1
|
|
await reportProgress("Briefpapiere importiert")
|
|
}
|
|
|
|
for (const table of regularTableNames) {
|
|
const rows = exportData.tables[table] || []
|
|
const metadata = columnsByTable.get(table)
|
|
if (!metadata) continue
|
|
|
|
const count = await insertRows(client, table, rows, metadata)
|
|
importedTables.push({ table, rows: count })
|
|
progressDone += 1
|
|
await reportProgress(`${table} importiert`)
|
|
}
|
|
|
|
const cleanedCommunicationRooms = await cleanupImportedCommunicationRooms(client, exportData)
|
|
if (cleanedCommunicationRooms) {
|
|
importedTables.push({ table: "communication_rooms_matrix_reset", rows: cleanedCommunicationRooms })
|
|
}
|
|
progressDone += 1
|
|
await reportProgress("Kommunikationsräume bereinigt")
|
|
|
|
await refreshSequences(client, columnsByTable)
|
|
progressDone = progressTotal
|
|
await reportProgress("Import abgeschlossen")
|
|
await client.query("commit")
|
|
|
|
return {
|
|
tenantId: exportData.tenantId,
|
|
tables: importedTables,
|
|
files,
|
|
}
|
|
} catch (err) {
|
|
await client.query("rollback")
|
|
throw err
|
|
} finally {
|
|
client.release()
|
|
}
|
|
}
|
|
|
|
export const importTenantFullExportArchive = async (
|
|
server: FastifyInstance,
|
|
archiveBuffer: Buffer,
|
|
options: ImportOptions = {}
|
|
): Promise<ImportResult> => {
|
|
const reader = new ZipReader(new BlobReader(new Blob([archiveBuffer])))
|
|
|
|
try {
|
|
const entries = await reader.getEntries()
|
|
const entriesByName = new Map(entries.map((entry: any) => [entry.filename, entry]))
|
|
const manifest = JSON.parse(await readZipTextEntry(entriesByName, "manifest.json")) as TenantArchiveManifest
|
|
|
|
if (manifest?.format !== "fedeo.tenant-archive-export" || manifest.version !== 1) {
|
|
throw new Error("Ungültiges FEDEO Mandantenarchiv-Format")
|
|
}
|
|
|
|
const tables: TableRows = {}
|
|
for (const table of manifest.tables || []) {
|
|
tables[table.name] = JSON.parse(await readZipTextEntry(entriesByName, table.path))
|
|
}
|
|
|
|
const rawExportData: TenantFullExport = {
|
|
format: "fedeo.tenant-full-export",
|
|
version: 1,
|
|
exportedAt: manifest.exportedAt,
|
|
tenantId: manifest.tenantId,
|
|
tables,
|
|
files: (manifest.files || []).map((file) => ({
|
|
id: file.id,
|
|
path: file.path,
|
|
name: file.name,
|
|
mimeType: file.mimeType,
|
|
size: file.size,
|
|
contentBase64: null,
|
|
missing: file.missing,
|
|
error: file.error,
|
|
})),
|
|
}
|
|
|
|
const exportData = remapTenantScopedExport(rawExportData, options.targetTenantId)
|
|
const sourceTenantId = rawExportData.tenantId
|
|
const targetTenantId = exportData.tenantId
|
|
const sourcePrefix = `${sourceTenantId}/`
|
|
const targetPrefix = `${targetTenantId}/`
|
|
const remappedManifest: TenantArchiveManifest = {
|
|
...manifest,
|
|
tenantId: targetTenantId,
|
|
files: (manifest.files || []).map((file) => ({
|
|
...file,
|
|
path: file.path?.startsWith(sourcePrefix)
|
|
? `${targetPrefix}${file.path.slice(sourcePrefix.length)}`
|
|
: file.path,
|
|
})),
|
|
}
|
|
|
|
const result = await importTenantFullExport(server, exportData, { targetTenantId: null })
|
|
const files = await restoreArchiveFiles(entriesByName, exportData, remappedManifest)
|
|
|
|
return {
|
|
...result,
|
|
files,
|
|
}
|
|
} finally {
|
|
await reader.close()
|
|
}
|
|
}
|