Restrict tenant imports to new tenants
This commit is contained in:
@@ -1,36 +0,0 @@
|
||||
type TableRows = Record<string, Record<string, any>[]>
|
||||
|
||||
type LoadRows = (
|
||||
table: string,
|
||||
whereSql: string,
|
||||
params?: any[]
|
||||
) => Promise<Record<string, any>[]>
|
||||
|
||||
type AddRows = (
|
||||
tables: TableRows,
|
||||
table: string,
|
||||
rows: Record<string, any>[]
|
||||
) => void
|
||||
|
||||
export const TENANT_EXPORT_GLOBAL_TABLES = ["units", "citys", "countrys"] as const
|
||||
|
||||
export const addTenantExportGlobalResources = async (
|
||||
tables: TableRows,
|
||||
availableTables: ReadonlySet<string>,
|
||||
tenant: Record<string, any>,
|
||||
loadRows: LoadRows,
|
||||
addRows: AddRows
|
||||
) => {
|
||||
for (const table of TENANT_EXPORT_GLOBAL_TABLES) {
|
||||
if (!availableTables.has(table)) continue
|
||||
addRows(tables, table, await loadRows(table, "true"))
|
||||
}
|
||||
|
||||
if (availableTables.has("accounts")) {
|
||||
addRows(
|
||||
tables,
|
||||
"accounts",
|
||||
await loadRows("accounts", `"accountChart" = $1`, [tenant.accountChart || "skr03"])
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,6 @@ import { pool } from "../../db"
|
||||
import { s3 } from "./s3"
|
||||
import { secrets } from "./secrets"
|
||||
import { decrypt, encrypt } from "./crypt"
|
||||
import { addTenantExportGlobalResources } from "./tenantExportGlobalResources"
|
||||
import { restoreImportedTenantNumberRanges } from "./tenantImportNumberRanges"
|
||||
|
||||
type TableRows = Record<string, Record<string, any>[]>
|
||||
type TableMetadata = {
|
||||
@@ -46,7 +44,6 @@ type ImportResult = {
|
||||
}
|
||||
|
||||
type ImportOptions = {
|
||||
targetTenantId?: number | null
|
||||
onProgress?: (progress: { done: number; total: number; message?: string }) => Promise<void> | void
|
||||
}
|
||||
|
||||
@@ -84,6 +81,10 @@ const ENTITY_BANKACCOUNT_PLAIN_FIELDS = {
|
||||
bankName: "__plainBankName",
|
||||
}
|
||||
|
||||
// Diese globalen Stammdaten werden installationsweit per SQL-Migration gepflegt
|
||||
// und duerfen auch aus aelteren Tenant-Archiven nicht importiert werden.
|
||||
const GLOBAL_MIGRATION_TABLES = new Set(["accounts", "units", "citys", "countrys"])
|
||||
|
||||
const quoteIdent = (value: string) => `"${value.replace(/"/g, '""')}"`
|
||||
const matrixServerName = () =>
|
||||
process.env.MATRIX_SERVER_NAME ||
|
||||
@@ -301,14 +302,6 @@ export const buildTenantFullExport = async (
|
||||
if (!tenantRows.length) throw new Error("Tenant nicht gefunden")
|
||||
|
||||
addRows(tables, "tenants", tenantRows)
|
||||
await addTenantExportGlobalResources(
|
||||
tables,
|
||||
new Set(columnsByTable.keys()),
|
||||
tenantRows[0],
|
||||
(table, whereSql, params) => loadRows(client, table, whereSql, params),
|
||||
addRows
|
||||
)
|
||||
|
||||
for (const [table, metadata] of columnsByTable.entries()) {
|
||||
if (table === "tenants") continue
|
||||
const { columns } = metadata
|
||||
@@ -612,58 +605,6 @@ const restoreArchiveFiles = async (
|
||||
return { restored, skipped }
|
||||
}
|
||||
|
||||
const remapTenantScopedExport = (
|
||||
exportData: TenantFullExport,
|
||||
targetTenantId?: number | null
|
||||
): TenantFullExport => {
|
||||
if (!targetTenantId || targetTenantId === exportData.tenantId) return exportData
|
||||
|
||||
const sourceTenantId = exportData.tenantId
|
||||
const sourcePathPrefix = `${sourceTenantId}/`
|
||||
const targetPathPrefix = `${targetTenantId}/`
|
||||
const tables: TableRows = {}
|
||||
|
||||
for (const [table, rows] of Object.entries(exportData.tables || {})) {
|
||||
tables[table] = rows.map((row) => {
|
||||
const nextRow = { ...row }
|
||||
|
||||
if (table === "tenants" && nextRow.id === sourceTenantId) {
|
||||
nextRow.id = targetTenantId
|
||||
}
|
||||
|
||||
if (nextRow.tenant === sourceTenantId) {
|
||||
nextRow.tenant = targetTenantId
|
||||
}
|
||||
|
||||
if (nextRow.tenant_id === sourceTenantId) {
|
||||
nextRow.tenant_id = targetTenantId
|
||||
}
|
||||
|
||||
if (table === "files" && typeof nextRow.path === "string" && nextRow.path.startsWith(sourcePathPrefix)) {
|
||||
nextRow.path = `${targetPathPrefix}${nextRow.path.slice(sourcePathPrefix.length)}`
|
||||
}
|
||||
|
||||
if (table === "letterheads" && typeof nextRow.path === "string" && nextRow.path.startsWith(sourcePathPrefix)) {
|
||||
nextRow.path = `${targetPathPrefix}${nextRow.path.slice(sourcePathPrefix.length)}`
|
||||
}
|
||||
|
||||
return nextRow
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
...exportData,
|
||||
tenantId: targetTenantId,
|
||||
tables,
|
||||
files: (exportData.files || []).map((file) => ({
|
||||
...file,
|
||||
path: file.path?.startsWith(sourcePathPrefix)
|
||||
? `${targetPathPrefix}${file.path.slice(sourcePathPrefix.length)}`
|
||||
: file.path,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
const encryptEntityBankAccountRowsForImport = (exportData: TenantFullExport) => {
|
||||
const rows = exportData.tables.entitybankaccounts || []
|
||||
|
||||
@@ -712,12 +653,6 @@ const prepareCommunicationRoomsForImport = (exportData: TenantFullExport) => {
|
||||
}
|
||||
}
|
||||
|
||||
export const prepareTenantFullExportRowsForImport = (exportData: TenantFullExport) => {
|
||||
encryptEntityBankAccountRowsForImport(exportData)
|
||||
prepareCommunicationRoomsForImport(exportData)
|
||||
return exportData
|
||||
}
|
||||
|
||||
const cleanupImportedCommunicationRooms = async (client: any, exportData: TenantFullExport) => {
|
||||
const rows = exportData.tables.communication_rooms || []
|
||||
if (!rows.length) return 0
|
||||
@@ -905,8 +840,9 @@ export const importTenantFullExport = async (
|
||||
throw new Error("Ungültiges FEDEO Mandantenexport-Format")
|
||||
}
|
||||
|
||||
const exportData = remapTenantScopedExport(rawExportData, options.targetTenantId)
|
||||
prepareTenantFullExportRowsForImport(exportData)
|
||||
const exportData = rawExportData
|
||||
encryptEntityBankAccountRowsForImport(exportData)
|
||||
prepareCommunicationRoomsForImport(exportData)
|
||||
const client = await pool.connect()
|
||||
const importOrder = [
|
||||
"tenants",
|
||||
@@ -921,11 +857,21 @@ export const importTenantFullExport = async (
|
||||
]
|
||||
|
||||
try {
|
||||
const existingTenant = await client.query(
|
||||
`select 1 from ${quoteIdent("tenants")} where ${quoteIdent("id")} = $1 limit 1`,
|
||||
[exportData.tenantId]
|
||||
)
|
||||
if (existingTenant.rows.length) {
|
||||
throw new Error("Ein Tenant mit dieser ID existiert bereits. Tenant-Exporte können nur als neuer Tenant importiert werden.")
|
||||
}
|
||||
|
||||
const columnsByTable = await tableColumns(client)
|
||||
const tableNames = [
|
||||
...importOrder,
|
||||
...Object.keys(exportData.tables).filter((table) => !importOrder.includes(table)).sort(),
|
||||
].filter((table, index, all) => all.indexOf(table) === index)
|
||||
].filter((table, index, all) =>
|
||||
all.indexOf(table) === index && !GLOBAL_MIGRATION_TABLES.has(table)
|
||||
)
|
||||
const specialTables = [
|
||||
columnsByTable.has("auth_tenant_users") ? "auth_tenant_users" : null,
|
||||
columnsByTable.has("bankaccounts") ? "bankaccounts" : null,
|
||||
@@ -1005,8 +951,6 @@ export const importTenantFullExport = async (
|
||||
progressDone += 1
|
||||
await reportProgress("Kommunikationsräume bereinigt")
|
||||
|
||||
await restoreImportedTenantNumberRanges(client, exportData)
|
||||
|
||||
await refreshSequences(client, columnsByTable)
|
||||
progressDone = progressTotal
|
||||
await reportProgress("Import abgeschlossen")
|
||||
@@ -1030,42 +974,6 @@ export const importTenantFullExportArchive = async (
|
||||
archiveBuffer: Buffer,
|
||||
options: ImportOptions = {}
|
||||
): Promise<ImportResult> => {
|
||||
const rawExportData = await readTenantFullExportArchive(archiveBuffer)
|
||||
const exportData = remapTenantScopedExport(rawExportData, options.targetTenantId)
|
||||
const reader = new ZipReader(new BlobReader(new Blob([archiveBuffer])))
|
||||
|
||||
try {
|
||||
const entries = await reader.getEntries()
|
||||
const entriesByName = new Map(entries.map((entry: any) => [entry.filename, entry]))
|
||||
const manifest = JSON.parse(await readZipTextEntry(entriesByName, "manifest.json")) as TenantArchiveManifest
|
||||
const sourceTenantId = rawExportData.tenantId
|
||||
const targetTenantId = exportData.tenantId
|
||||
const sourcePrefix = `${sourceTenantId}/`
|
||||
const targetPrefix = `${targetTenantId}/`
|
||||
const remappedManifest: TenantArchiveManifest = {
|
||||
...manifest,
|
||||
tenantId: targetTenantId,
|
||||
files: (manifest.files || []).map((file) => ({
|
||||
...file,
|
||||
path: file.path?.startsWith(sourcePrefix)
|
||||
? `${targetPrefix}${file.path.slice(sourcePrefix.length)}`
|
||||
: file.path,
|
||||
})),
|
||||
}
|
||||
|
||||
const result = await importTenantFullExport(server, exportData, { targetTenantId: null })
|
||||
const files = await restoreArchiveFiles(entriesByName, exportData, remappedManifest)
|
||||
|
||||
return {
|
||||
...result,
|
||||
files,
|
||||
}
|
||||
} finally {
|
||||
await reader.close()
|
||||
}
|
||||
}
|
||||
|
||||
export const readTenantFullExportArchive = async (archiveBuffer: Buffer): Promise<TenantFullExport> => {
|
||||
const reader = new ZipReader(new BlobReader(new Blob([archiveBuffer])))
|
||||
|
||||
try {
|
||||
@@ -1082,7 +990,7 @@ export const readTenantFullExportArchive = async (archiveBuffer: Buffer): Promis
|
||||
tables[table.name] = JSON.parse(await readZipTextEntry(entriesByName, table.path))
|
||||
}
|
||||
|
||||
return {
|
||||
const rawExportData: TenantFullExport = {
|
||||
format: "fedeo.tenant-full-export",
|
||||
version: 1,
|
||||
exportedAt: manifest.exportedAt,
|
||||
@@ -1100,57 +1008,14 @@ export const readTenantFullExportArchive = async (archiveBuffer: Buffer): Promis
|
||||
})),
|
||||
}
|
||||
|
||||
const result = await importTenantFullExport(server, rawExportData)
|
||||
const files = await restoreArchiveFiles(entriesByName, rawExportData, manifest)
|
||||
|
||||
return {
|
||||
...result,
|
||||
files,
|
||||
}
|
||||
} finally {
|
||||
await reader.close()
|
||||
}
|
||||
}
|
||||
|
||||
export const restoreTenantMergeArchiveFiles = async (
|
||||
archiveBuffer: Buffer,
|
||||
rawExportData: TenantFullExport,
|
||||
targetTenantId: number,
|
||||
selectedFileRefs: ReadonlySet<string>
|
||||
) => {
|
||||
const reader = new ZipReader(new BlobReader(new Blob([archiveBuffer])))
|
||||
try {
|
||||
const entries = await reader.getEntries()
|
||||
const entriesByName = new Map(entries.map((entry: any) => [entry.filename, entry]))
|
||||
const manifest = JSON.parse(await readZipTextEntry(entriesByName, "manifest.json")) as TenantArchiveManifest
|
||||
const exportData = remapTenantScopedExport(rawExportData, targetTenantId)
|
||||
const sourcePrefix = `${rawExportData.tenantId}/`
|
||||
const targetPrefix = `${targetTenantId}/`
|
||||
const filteredFiles = (manifest.files || [])
|
||||
.filter((file) => selectedFileRefs.has(String(file.id)))
|
||||
.map((file) => ({
|
||||
...file,
|
||||
path: file.path?.startsWith(sourcePrefix)
|
||||
? `${targetPrefix}${file.path.slice(sourcePrefix.length)}`
|
||||
: file.path,
|
||||
}))
|
||||
const filteredManifest: TenantArchiveManifest = {
|
||||
...manifest,
|
||||
tenantId: targetTenantId,
|
||||
files: filteredFiles,
|
||||
}
|
||||
const filteredExportData: TenantFullExport = {
|
||||
...exportData,
|
||||
files: exportData.files.filter((file) => selectedFileRefs.has(String(file.id))),
|
||||
}
|
||||
|
||||
return await restoreArchiveFiles(entriesByName, filteredExportData, filteredManifest)
|
||||
} finally {
|
||||
await reader.close()
|
||||
}
|
||||
}
|
||||
|
||||
export const restoreTenantMergeInlineFiles = async (
|
||||
rawExportData: TenantFullExport,
|
||||
targetTenantId: number,
|
||||
selectedFileRefs: ReadonlySet<string>
|
||||
) => {
|
||||
const exportData = remapTenantScopedExport(rawExportData, targetTenantId)
|
||||
return await restoreFiles({
|
||||
...exportData,
|
||||
files: exportData.files.filter((file) => selectedFileRefs.has(String(file.id))),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
type QueryClient = {
|
||||
query: (query: string, values: unknown[]) => Promise<{ rowCount?: number | null }>
|
||||
}
|
||||
|
||||
type TenantImportData = {
|
||||
tenantId: number
|
||||
tables: Record<string, Record<string, any>[]>
|
||||
}
|
||||
|
||||
export const restoreImportedTenantNumberRanges = async (
|
||||
client: QueryClient,
|
||||
exportData: TenantImportData
|
||||
) => {
|
||||
const tenantRow = (exportData.tables.tenants || []).find(
|
||||
(row) => Number(row.id) === Number(exportData.tenantId)
|
||||
)
|
||||
|
||||
if (!tenantRow || tenantRow.numberRanges === null || typeof tenantRow.numberRanges === "undefined") {
|
||||
return 0
|
||||
}
|
||||
|
||||
const result = await client.query(
|
||||
`update "tenants" set "numberRanges" = $1::jsonb where "id" = $2`,
|
||||
[JSON.stringify(tenantRow.numberRanges), exportData.tenantId]
|
||||
)
|
||||
|
||||
return result.rowCount || 0
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
import { createHash } from "crypto"
|
||||
|
||||
export type TenantMergeKind = "import" | "existing" | "conflict" | "id_collision"
|
||||
export type TenantMergeDecision = "source" | "target"
|
||||
|
||||
export type TenantMergeTableMetadata = {
|
||||
primaryKey: string[]
|
||||
}
|
||||
|
||||
export type TenantMergeItem = {
|
||||
id: string
|
||||
table: string
|
||||
kind: TenantMergeKind
|
||||
sourceKey: string
|
||||
targetKey: string | null
|
||||
label: string
|
||||
defaultDecision: TenantMergeDecision
|
||||
allowedDecisions: TenantMergeDecision[]
|
||||
differences: string[]
|
||||
sourceRow: Record<string, any>
|
||||
targetRow: Record<string, any> | null
|
||||
}
|
||||
|
||||
export type TenantMergePlan = {
|
||||
items: TenantMergeItem[]
|
||||
summary: Record<TenantMergeKind, number>
|
||||
}
|
||||
|
||||
const ignoredComparisonColumns = new Set([
|
||||
"created_at",
|
||||
"createdAt",
|
||||
"updated_at",
|
||||
"updatedAt",
|
||||
"updated_by",
|
||||
"updatedBy",
|
||||
"locked",
|
||||
"locked_by_export_job_id",
|
||||
"lockedByExportJobId",
|
||||
])
|
||||
|
||||
const normalizeText = (value: unknown) => String(value ?? "").trim().toLocaleLowerCase("de")
|
||||
|
||||
const naturalKeyColumns: Record<string, string[]> = {
|
||||
accounts: ["accountChart", "number"],
|
||||
units: ["name"],
|
||||
countrys: ["name"],
|
||||
citys: ["zip", "short", "districtCode"],
|
||||
}
|
||||
|
||||
const comparableRow = (row: Record<string, any>) => Object.fromEntries(
|
||||
Object.entries(row)
|
||||
.filter(([column]) => !ignoredComparisonColumns.has(column))
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
)
|
||||
|
||||
const rowDifferences = (table: string, source: Record<string, any>, target: Record<string, any>) => {
|
||||
const sourceComparable = comparableRow(source)
|
||||
const targetComparable = comparableRow(target)
|
||||
if (table === "citys") {
|
||||
delete sourceComparable.geometry
|
||||
delete targetComparable.geometry
|
||||
}
|
||||
const columns = Array.from(new Set([
|
||||
...Object.keys(sourceComparable),
|
||||
...Object.keys(targetComparable),
|
||||
])).sort()
|
||||
|
||||
return columns.filter((column) =>
|
||||
JSON.stringify(sourceComparable[column]) !== JSON.stringify(targetComparable[column])
|
||||
)
|
||||
}
|
||||
|
||||
const rowKey = (row: Record<string, any>, columns: string[]) =>
|
||||
columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join("|")
|
||||
|
||||
const naturalKey = (table: string, row: Record<string, any>) => {
|
||||
const columns = naturalKeyColumns[table]
|
||||
if (!columns) return null
|
||||
|
||||
const values = columns.map((column) => row[column])
|
||||
if (values.every((value) => value === null || typeof value === "undefined" || value === "")) return null
|
||||
|
||||
return columns.map((column, index) => `${column}=${normalizeText(values[index])}`).join("|")
|
||||
}
|
||||
|
||||
const itemId = (table: string, sourceKey: string) =>
|
||||
createHash("sha256").update(`${table}\0${sourceKey}`).digest("hex").slice(0, 24)
|
||||
|
||||
const itemLabel = (table: string, row: Record<string, any>, fallback: string) => {
|
||||
const descriptive = row.name ?? row.label ?? row.title ?? row.number ?? row.email ?? row.filename
|
||||
return descriptive ? `${descriptive}` : `${table}: ${fallback}`
|
||||
}
|
||||
|
||||
export const buildTenantMergePlan = (
|
||||
sourceTables: Record<string, Record<string, any>[]>,
|
||||
targetTables: Record<string, Record<string, any>[]>,
|
||||
metadata: Record<string, TenantMergeTableMetadata>
|
||||
): TenantMergePlan => {
|
||||
const items: TenantMergeItem[] = []
|
||||
|
||||
for (const table of Object.keys(sourceTables).sort()) {
|
||||
const primaryKey = metadata[table]?.primaryKey?.length ? metadata[table].primaryKey : ["id"]
|
||||
const targetRows = targetTables[table] || []
|
||||
const targetByPrimaryKey = new Map(targetRows.map((row) => [rowKey(row, primaryKey), row]))
|
||||
const targetByNaturalKey = new Map<string, Record<string, any>>()
|
||||
|
||||
for (const targetRow of targetRows) {
|
||||
const key = naturalKey(table, targetRow)
|
||||
if (key) targetByNaturalKey.set(key, targetRow)
|
||||
}
|
||||
|
||||
for (const sourceRow of sourceTables[table] || []) {
|
||||
const sourceKey = rowKey(sourceRow, primaryKey)
|
||||
const sourceNaturalKey = naturalKey(table, sourceRow)
|
||||
const naturalTarget = sourceNaturalKey ? targetByNaturalKey.get(sourceNaturalKey) : undefined
|
||||
const primaryTarget = targetByPrimaryKey.get(sourceKey)
|
||||
const targetRow = naturalTarget || primaryTarget || null
|
||||
const differences = targetRow ? rowDifferences(table, sourceRow, targetRow) : []
|
||||
let kind: TenantMergeKind
|
||||
|
||||
if (!targetRow) {
|
||||
kind = "import"
|
||||
} else if (!differences.length) {
|
||||
kind = "existing"
|
||||
} else if (primaryTarget && sourceNaturalKey && naturalKey(table, primaryTarget) !== sourceNaturalKey) {
|
||||
kind = "id_collision"
|
||||
} else {
|
||||
kind = "conflict"
|
||||
}
|
||||
|
||||
items.push({
|
||||
id: itemId(table, sourceKey),
|
||||
table,
|
||||
kind,
|
||||
sourceKey,
|
||||
targetKey: targetRow ? rowKey(targetRow, primaryKey) : null,
|
||||
label: itemLabel(table, sourceRow, sourceKey),
|
||||
defaultDecision: kind === "import" || kind === "id_collision" ? "source" : "target",
|
||||
allowedDecisions: kind === "conflict" ? ["target", "source"] : [kind === "existing" ? "target" : "source"],
|
||||
differences,
|
||||
sourceRow,
|
||||
targetRow,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
items,
|
||||
summary: {
|
||||
import: items.filter((item) => item.kind === "import").length,
|
||||
existing: items.filter((item) => item.kind === "existing").length,
|
||||
conflict: items.filter((item) => item.kind === "conflict").length,
|
||||
id_collision: items.filter((item) => item.kind === "id_collision").length,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,438 +0,0 @@
|
||||
import { pool } from "../../db"
|
||||
import { prepareTenantFullExportRowsForImport, type TenantFullExport } from "./tenantFullExport"
|
||||
import { buildTenantMergePlan, type TenantMergeDecision, type TenantMergePlan, type TenantMergeTableMetadata } from "./tenantMergePlan"
|
||||
|
||||
type MergeDatabaseMetadata = TenantMergeTableMetadata & {
|
||||
columns: string[]
|
||||
jsonColumns: Set<string>
|
||||
generatedColumns: Set<string>
|
||||
foreignKeys: { column: string, referencedTable: string, referencedColumn: string }[]
|
||||
}
|
||||
|
||||
const quoteIdent = (value: string) => `"${value.replace(/"/g, '""')}"`
|
||||
const globalNaturalKeyTables = new Set(["accounts", "units", "countrys", "citys"])
|
||||
|
||||
const loadMergeMetadata = async (client: any) => {
|
||||
const columnsResult = await client.query(`
|
||||
select table_name, column_name, data_type, is_generated
|
||||
from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
order by table_name, ordinal_position
|
||||
`)
|
||||
const foreignKeyResult = await client.query(`
|
||||
select tc.table_name, kcu.column_name, ccu.table_name as referenced_table,
|
||||
ccu.column_name as referenced_column
|
||||
from information_schema.table_constraints tc
|
||||
join information_schema.key_column_usage kcu
|
||||
on tc.constraint_name = kcu.constraint_name
|
||||
and tc.constraint_schema = kcu.constraint_schema
|
||||
join information_schema.constraint_column_usage ccu
|
||||
on tc.constraint_name = ccu.constraint_name
|
||||
and tc.constraint_schema = ccu.constraint_schema
|
||||
where tc.table_schema = 'public' and tc.constraint_type = 'FOREIGN KEY'
|
||||
`)
|
||||
const primaryKeyResult = await client.query(`
|
||||
select tc.table_name, kcu.column_name
|
||||
from information_schema.table_constraints tc
|
||||
join information_schema.key_column_usage kcu
|
||||
on tc.constraint_name = kcu.constraint_name
|
||||
and tc.constraint_schema = kcu.constraint_schema
|
||||
where tc.table_schema = 'public' and tc.constraint_type = 'PRIMARY KEY'
|
||||
order by tc.table_name, kcu.ordinal_position
|
||||
`)
|
||||
const metadata: Record<string, MergeDatabaseMetadata> = {}
|
||||
|
||||
for (const row of columnsResult.rows) {
|
||||
metadata[row.table_name] ||= { columns: [], primaryKey: [], jsonColumns: new Set(), generatedColumns: new Set(), foreignKeys: [] }
|
||||
metadata[row.table_name].columns.push(row.column_name)
|
||||
if (row.data_type === "json" || row.data_type === "jsonb") metadata[row.table_name].jsonColumns.add(row.column_name)
|
||||
if (row.is_generated === "ALWAYS") metadata[row.table_name].generatedColumns.add(row.column_name)
|
||||
}
|
||||
for (const row of primaryKeyResult.rows) {
|
||||
metadata[row.table_name] ||= { columns: [], primaryKey: [], jsonColumns: new Set(), generatedColumns: new Set(), foreignKeys: [] }
|
||||
metadata[row.table_name].primaryKey.push(row.column_name)
|
||||
}
|
||||
for (const row of foreignKeyResult.rows) {
|
||||
metadata[row.table_name] ||= { columns: [], primaryKey: [], jsonColumns: new Set(), generatedColumns: new Set(), foreignKeys: [] }
|
||||
metadata[row.table_name].foreignKeys.push({
|
||||
column: row.column_name,
|
||||
referencedTable: row.referenced_table,
|
||||
referencedColumn: row.referenced_column,
|
||||
})
|
||||
}
|
||||
|
||||
return metadata
|
||||
}
|
||||
|
||||
const remapSourceTenant = (exportData: TenantFullExport, targetTenantId: number) => {
|
||||
const tables: Record<string, Record<string, any>[]> = {}
|
||||
|
||||
for (const [table, rows] of Object.entries(exportData.tables || {})) {
|
||||
tables[table] = rows.map((row) => {
|
||||
const next = { ...row }
|
||||
if (table === "tenants" && Number(next.id) === Number(exportData.tenantId)) next.id = targetTenantId
|
||||
if (Number(next.tenant) === Number(exportData.tenantId)) next.tenant = targetTenantId
|
||||
if (Number(next.tenant_id) === Number(exportData.tenantId)) next.tenant_id = targetTenantId
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
const splitSourceTablesByAvailability = (
|
||||
sourceTables: Record<string, Record<string, any>[]>,
|
||||
metadata: Record<string, MergeDatabaseMetadata>
|
||||
) => {
|
||||
const available: Record<string, Record<string, any>[]> = {}
|
||||
const unavailable = Object.keys(sourceTables).filter((table) => {
|
||||
if (metadata[table]) {
|
||||
available[table] = sourceTables[table]
|
||||
return false
|
||||
}
|
||||
return sourceTables[table].length > 0
|
||||
})
|
||||
|
||||
return { available, unavailable }
|
||||
}
|
||||
|
||||
const appendUnavailableTableConflicts = (plan: TenantMergePlan, unavailableTables: string[]) => {
|
||||
for (const table of unavailableTables.sort()) {
|
||||
plan.items.push({
|
||||
id: `missing-target-table:${table}`,
|
||||
table,
|
||||
kind: "conflict",
|
||||
sourceKey: "Zieltabelle fehlt",
|
||||
targetKey: null,
|
||||
label: `Tabelle ${table} ist im Zielsystem nicht vorhanden`,
|
||||
defaultDecision: "target",
|
||||
allowedDecisions: ["target"],
|
||||
differences: ["Die erforderliche Tabelle fehlt im Zielschema und muss zuerst per Migration angelegt werden."],
|
||||
sourceRow: {},
|
||||
targetRow: null,
|
||||
})
|
||||
plan.summary.conflict += 1
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
const loadTargetRows = async (
|
||||
client: any,
|
||||
table: string,
|
||||
sourceRows: Record<string, any>[],
|
||||
metadata: MergeDatabaseMetadata,
|
||||
targetTenantId: number
|
||||
) => {
|
||||
if (!sourceRows.length) return []
|
||||
if (table === "tenants") {
|
||||
return (await client.query(`select * from ${quoteIdent(table)} where ${quoteIdent("id")} = $1`, [targetTenantId])).rows
|
||||
}
|
||||
|
||||
const tenantColumn = metadata.columns.includes("tenant")
|
||||
? "tenant"
|
||||
: metadata.columns.includes("tenant_id") ? "tenant_id" : null
|
||||
if (tenantColumn) {
|
||||
return (await client.query(
|
||||
`select * from ${quoteIdent(table)} where ${quoteIdent(tenantColumn)} = $1`,
|
||||
[targetTenantId]
|
||||
)).rows
|
||||
}
|
||||
if (globalNaturalKeyTables.has(table)) {
|
||||
return (await client.query(`select * from ${quoteIdent(table)}`)).rows
|
||||
}
|
||||
|
||||
if (metadata.primaryKey.length === 1) {
|
||||
const key = metadata.primaryKey[0]
|
||||
const values = Array.from(new Set(sourceRows.map((row) => row[key]).filter((value) => value !== null && typeof value !== "undefined").map(String)))
|
||||
if (!values.length) return []
|
||||
return (await client.query(
|
||||
`select * from ${quoteIdent(table)} where ${quoteIdent(key)}::text = any($1::text[])`,
|
||||
[values]
|
||||
)).rows
|
||||
}
|
||||
|
||||
return (await client.query(`select * from ${quoteIdent(table)}`)).rows
|
||||
}
|
||||
|
||||
export const createTenantMergeDryRunWithClient = async (
|
||||
client: any,
|
||||
exportData: TenantFullExport,
|
||||
targetTenantId: number
|
||||
): Promise<TenantMergePlan> => {
|
||||
const metadata = await loadMergeMetadata(client)
|
||||
const remappedSourceTables = remapSourceTenant(exportData, targetTenantId)
|
||||
const { available: sourceTables, unavailable } = splitSourceTablesByAvailability(remappedSourceTables, metadata)
|
||||
const targetTables: Record<string, Record<string, any>[]> = {}
|
||||
|
||||
for (const [table, sourceRows] of Object.entries(sourceTables)) {
|
||||
const tableMetadata = metadata[table]
|
||||
if (!tableMetadata) continue
|
||||
targetTables[table] = await loadTargetRows(client, table, sourceRows, tableMetadata, targetTenantId)
|
||||
}
|
||||
|
||||
return appendUnavailableTableConflicts(buildTenantMergePlan(sourceTables, targetTables, metadata), unavailable)
|
||||
}
|
||||
|
||||
export const createTenantMergeDryRun = async (
|
||||
exportData: TenantFullExport,
|
||||
targetTenantId: number
|
||||
) => {
|
||||
const client = await pool.connect()
|
||||
try {
|
||||
return await createTenantMergeDryRunWithClient(client, exportData, targetTenantId)
|
||||
} finally {
|
||||
client.release()
|
||||
}
|
||||
}
|
||||
|
||||
const sensitiveColumns = new Set([
|
||||
"iban_encrypted",
|
||||
"bic_encrypted",
|
||||
"bank_name_encrypted",
|
||||
"__plainIban",
|
||||
"__plainBic",
|
||||
"__plainBankName",
|
||||
"password_hash",
|
||||
"passwordHash",
|
||||
"token_hash",
|
||||
"tokenHash",
|
||||
])
|
||||
|
||||
const compactValue = (value: any) => {
|
||||
if (value === null || typeof value !== "object") return value
|
||||
const serialized = JSON.stringify(value)
|
||||
return `[Struktur mit ${serialized.length} Zeichen]`
|
||||
}
|
||||
|
||||
const compactReviewRow = (row: Record<string, any> | null, differences: string[]) => {
|
||||
if (!row) return null
|
||||
const displayColumns = new Set([
|
||||
"id",
|
||||
"tenant",
|
||||
"tenant_id",
|
||||
"name",
|
||||
"label",
|
||||
"title",
|
||||
"number",
|
||||
"email",
|
||||
"zip",
|
||||
"short",
|
||||
"accountChart",
|
||||
...differences,
|
||||
])
|
||||
|
||||
return Object.fromEntries(Object.entries(row)
|
||||
.filter(([column]) => displayColumns.has(column))
|
||||
.map(([column, value]) => [
|
||||
column,
|
||||
sensitiveColumns.has(column) && value ? "***" : compactValue(value),
|
||||
]))
|
||||
}
|
||||
|
||||
export const sanitizeTenantMergePlan = (plan: TenantMergePlan): TenantMergePlan => ({
|
||||
...plan,
|
||||
items: plan.items.filter((item) => item.kind !== "existing").map((item) => ({
|
||||
...item,
|
||||
sourceRow: compactReviewRow(item.sourceRow, item.differences) || {},
|
||||
targetRow: compactReviewRow(item.targetRow, item.differences),
|
||||
})),
|
||||
})
|
||||
|
||||
const prepareValue = (value: any, isJson: boolean) => {
|
||||
if (!isJson || value === null || typeof value === "undefined" || typeof value === "string") return value
|
||||
return JSON.stringify(value)
|
||||
}
|
||||
|
||||
const rowIdentity = (row: Record<string, any>, columns: string[]) =>
|
||||
columns.map((column) => row[column]).join("\0")
|
||||
|
||||
const topologicalTableOrder = (tables: string[], metadata: Record<string, MergeDatabaseMetadata>) => {
|
||||
const remaining = new Set(tables)
|
||||
const ordered: string[] = []
|
||||
while (remaining.size) {
|
||||
const ready = Array.from(remaining).filter((table) =>
|
||||
(metadata[table]?.foreignKeys || []).every((foreignKey) =>
|
||||
!remaining.has(foreignKey.referencedTable) || foreignKey.referencedTable === table
|
||||
)
|
||||
)
|
||||
const next = ready.length ? ready.sort() : [Array.from(remaining).sort()[0]]
|
||||
for (const table of next) {
|
||||
remaining.delete(table)
|
||||
ordered.push(table)
|
||||
}
|
||||
}
|
||||
return ordered
|
||||
}
|
||||
|
||||
export type TenantMergeExecutionResult = {
|
||||
imported: number
|
||||
updated: number
|
||||
retained: number
|
||||
remappedIds: number
|
||||
tables: Record<string, { imported: number, updated: number, retained: number }>
|
||||
}
|
||||
|
||||
export const executeTenantMergeWithClient = async (
|
||||
client: any,
|
||||
rawExportData: TenantFullExport,
|
||||
targetTenantId: number,
|
||||
decisions: Record<string, TenantMergeDecision>
|
||||
): Promise<TenantMergeExecutionResult> => {
|
||||
const metadata = await loadMergeMetadata(client)
|
||||
const remappedSourceTables = remapSourceTenant(rawExportData, targetTenantId)
|
||||
const { available: sourceTables } = splitSourceTablesByAvailability(remappedSourceTables, metadata)
|
||||
const preparedExport: TenantFullExport = prepareTenantFullExportRowsForImport({
|
||||
...rawExportData,
|
||||
tenantId: targetTenantId,
|
||||
tables: Object.fromEntries(Object.entries(sourceTables).map(([table, rows]) => [table, rows.map((row) => ({ ...row }))])),
|
||||
})
|
||||
const targetTables: Record<string, Record<string, any>[]> = {}
|
||||
for (const [table, rows] of Object.entries(preparedExport.tables)) {
|
||||
if (metadata[table]) targetTables[table] = await loadTargetRows(client, table, rows, metadata[table], targetTenantId)
|
||||
}
|
||||
const plan = buildTenantMergePlan(preparedExport.tables, targetTables, metadata)
|
||||
const idMaps = new Map<string, Map<any, any>>()
|
||||
const selected = plan.items.filter((item) => {
|
||||
const decision = decisions[item.id] || item.defaultDecision
|
||||
return decision === "source" && item.kind !== "existing"
|
||||
})
|
||||
|
||||
for (const item of plan.items) {
|
||||
const tableMetadata = metadata[item.table]
|
||||
if (tableMetadata?.primaryKey.length !== 1 || !item.targetRow) continue
|
||||
const key = tableMetadata.primaryKey[0]
|
||||
const sourceId = item.sourceRow[key]
|
||||
const targetId = item.targetRow[key]
|
||||
if (sourceId !== null && typeof sourceId !== "undefined" && targetId !== null && typeof targetId !== "undefined") {
|
||||
if (!idMaps.has(item.table)) idMaps.set(item.table, new Map())
|
||||
idMaps.get(item.table)!.set(sourceId, targetId)
|
||||
}
|
||||
}
|
||||
|
||||
await client.query("begin")
|
||||
await client.query("set local session_replication_role = replica")
|
||||
try {
|
||||
for (const item of selected.filter((entry) => entry.kind === "id_collision")) {
|
||||
const tableMetadata = metadata[item.table]
|
||||
if (tableMetadata.primaryKey.length !== 1) throw new Error(`ID-Kollision in ${item.table} kann nicht automatisch aufgelöst werden`)
|
||||
const key = tableMetadata.primaryKey[0]
|
||||
const sequenceResult = await client.query("select pg_get_serial_sequence($1, $2) as sequence_name", [`public.${item.table}`, key])
|
||||
const sequenceName = sequenceResult.rows[0]?.sequence_name
|
||||
if (!sequenceName) throw new Error(`Keine Sequenz für ID-Kollision in ${item.table}.${key} gefunden`)
|
||||
const allocated = await client.query("select nextval($1::regclass) as id", [sequenceName])
|
||||
const sourceId = item.sourceRow[key]
|
||||
const targetId = allocated.rows[0].id
|
||||
item.sourceRow[key] = targetId
|
||||
if (!idMaps.has(item.table)) idMaps.set(item.table, new Map())
|
||||
idMaps.get(item.table)!.set(sourceId, targetId)
|
||||
}
|
||||
|
||||
for (const item of selected) {
|
||||
const tableMetadata = metadata[item.table]
|
||||
for (const foreignKey of tableMetadata.foreignKeys) {
|
||||
const mapping = idMaps.get(foreignKey.referencedTable)
|
||||
if (mapping?.has(item.sourceRow[foreignKey.column])) {
|
||||
item.sourceRow[foreignKey.column] = mapping.get(item.sourceRow[foreignKey.column])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const selectedByTable = new Map<string, typeof selected>()
|
||||
for (const item of selected) {
|
||||
const rows = selectedByTable.get(item.table) || []
|
||||
rows.push(item)
|
||||
selectedByTable.set(item.table, rows)
|
||||
}
|
||||
const result: TenantMergeExecutionResult = {
|
||||
imported: 0,
|
||||
updated: 0,
|
||||
retained: plan.items.length - selected.length,
|
||||
remappedIds: Array.from(idMaps.values()).reduce((sum, map) => sum + Array.from(map).filter(([source, target]) => source !== target).length, 0),
|
||||
tables: {},
|
||||
}
|
||||
|
||||
for (const table of topologicalTableOrder(Array.from(selectedByTable.keys()), metadata)) {
|
||||
const tableMetadata = metadata[table]
|
||||
result.tables[table] ||= { imported: 0, updated: 0, retained: plan.items.filter((item) => item.table === table && !selected.includes(item)).length }
|
||||
for (const item of selectedByTable.get(table) || []) {
|
||||
const row = { ...item.sourceRow }
|
||||
if (table === "tenants") {
|
||||
delete row.locked
|
||||
delete row.locked_by_export_job_id
|
||||
delete row.lockedByExportJobId
|
||||
}
|
||||
const columns = Object.keys(row).filter((column) =>
|
||||
tableMetadata.columns.includes(column) && !tableMetadata.generatedColumns.has(column)
|
||||
)
|
||||
const values = columns.map((column) => prepareValue(row[column], tableMetadata.jsonColumns.has(column)))
|
||||
const placeholders = columns.map((_, index) => `$${index + 1}`).join(", ")
|
||||
|
||||
if (item.kind === "conflict" && item.targetRow) {
|
||||
const primaryKey = tableMetadata.primaryKey
|
||||
const updateColumns = columns.filter((column) => !primaryKey.includes(column))
|
||||
if (!primaryKey.length || !updateColumns.length) continue
|
||||
const whereValues = primaryKey.map((column) => item.targetRow![column])
|
||||
const assignments = updateColumns.map((column) => `${quoteIdent(column)} = $${columns.indexOf(column) + 1}`).join(", ")
|
||||
const where = primaryKey.map((column, index) => `${quoteIdent(column)} = $${columns.length + index + 1}`).join(" and ")
|
||||
await client.query(`update ${quoteIdent(table)} set ${assignments} where ${where}`, [...values, ...whereValues])
|
||||
result.updated += 1
|
||||
result.tables[table].updated += 1
|
||||
} else {
|
||||
const inserted = await client.query(
|
||||
`insert into ${quoteIdent(table)} (${columns.map(quoteIdent).join(", ")}) values (${placeholders}) on conflict do nothing`,
|
||||
values
|
||||
)
|
||||
if (!inserted.rowCount) throw new Error(`Datensatz in ${table} konnte wegen eines neuen Konflikts nicht importiert werden`)
|
||||
result.imported += 1
|
||||
result.tables[table].imported += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sourceTenant = preparedExport.tables.tenants?.find((row) => Number(row.id) === targetTenantId)
|
||||
if (sourceTenant?.numberRanges) {
|
||||
const targetTenantResult = await client.query(`select "numberRanges" from "tenants" where "id" = $1`, [targetTenantId])
|
||||
const targetRanges = targetTenantResult.rows[0]?.numberRanges || {}
|
||||
const mergedRanges = { ...targetRanges }
|
||||
for (const [key, sourceRange] of Object.entries(sourceTenant.numberRanges as Record<string, any>)) {
|
||||
const targetRange = targetRanges[key]
|
||||
mergedRanges[key] = targetRange
|
||||
? {
|
||||
...sourceRange,
|
||||
...targetRange,
|
||||
nextNumber: Math.max(Number(sourceRange?.nextNumber || 0), Number(targetRange?.nextNumber || 0)),
|
||||
}
|
||||
: sourceRange
|
||||
}
|
||||
await client.query(`update "tenants" set "numberRanges" = $1::jsonb where "id" = $2`, [JSON.stringify(mergedRanges), targetTenantId])
|
||||
}
|
||||
|
||||
for (const table of selectedByTable.keys()) {
|
||||
const tableMetadata = metadata[table]
|
||||
if (!tableMetadata.columns.includes("id")) continue
|
||||
const sequenceResult = await client.query("select pg_get_serial_sequence($1, $2) as sequence_name", [`public.${table}`, "id"])
|
||||
const sequenceName = sequenceResult.rows[0]?.sequence_name
|
||||
if (!sequenceName) continue
|
||||
await client.query(`select setval($1::regclass, greatest(coalesce((select max(id) from ${quoteIdent(table)}), 1), 1), true)`, [sequenceName])
|
||||
}
|
||||
|
||||
await client.query("commit")
|
||||
return result
|
||||
} catch (err) {
|
||||
await client.query("rollback")
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export const executeTenantMerge = async (
|
||||
exportData: TenantFullExport,
|
||||
targetTenantId: number,
|
||||
decisions: Record<string, TenantMergeDecision>
|
||||
) => {
|
||||
const client = await pool.connect()
|
||||
try {
|
||||
return await executeTenantMergeWithClient(client, exportData, targetTenantId, decisions)
|
||||
} finally {
|
||||
client.release()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user