Fixes
This commit is contained in:
270
backend/scripts/import-members-csv.ts
Normal file
270
backend/scripts/import-members-csv.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
import fs from "node:fs"
|
||||
import path from "node:path"
|
||||
import { and, eq } from "drizzle-orm"
|
||||
|
||||
import { db, pool } from "../db"
|
||||
import { customers, entitybankaccounts } from "../db/schema"
|
||||
import { decrypt, encrypt } from "../src/utils/crypt"
|
||||
import { loadSecrets, secrets } from "../src/utils/secrets"
|
||||
|
||||
type CsvMemberRow = {
|
||||
number: string
|
||||
lastname: string
|
||||
firstname: string
|
||||
street: string
|
||||
zip: string
|
||||
city: string
|
||||
birthdate: string
|
||||
mobile: string
|
||||
email: string
|
||||
bankInstitute: string
|
||||
iban: string
|
||||
bic: string
|
||||
date: string
|
||||
memberStatus: string
|
||||
}
|
||||
|
||||
const TENANT_ID = 38
|
||||
const DEFAULT_CSV_PATH = "/Users/florianfederspiel/Downloads/Mitglieder Übersicht 2026_1.csv"
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const dryRun = args.includes("--dry-run")
|
||||
const csvArg = args.find((arg) => !arg.startsWith("--"))
|
||||
const csvPath = csvArg || DEFAULT_CSV_PATH
|
||||
|
||||
function normalizeIban(value: string) {
|
||||
return String(value || "").replace(/\s+/g, "").toUpperCase()
|
||||
}
|
||||
|
||||
function parseGermanDate(value: string): string | null {
|
||||
const v = String(value || "").trim()
|
||||
if (!v) return null
|
||||
|
||||
const m = v.match(/^(\d{1,2})\.(\d{1,2})\.(\d{2}|\d{4})$/)
|
||||
if (!m) return null
|
||||
|
||||
const day = m[1].padStart(2, "0")
|
||||
const month = m[2].padStart(2, "0")
|
||||
const yy = m[3]
|
||||
const year = yy.length === 4 ? yy : (Number(yy) >= 70 ? `19${yy}` : `20${yy}`)
|
||||
|
||||
return `${year}-${month}-${day}`
|
||||
}
|
||||
|
||||
function parseBoolFromStatus(value: string) {
|
||||
const normalized = String(value || "").trim().toLowerCase()
|
||||
return normalized !== "inaktiv"
|
||||
}
|
||||
|
||||
function parseCsv(content: string): CsvMemberRow[] {
|
||||
const lines = content
|
||||
.split(/\r?\n/)
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l.length > 0)
|
||||
|
||||
if (!lines.length) return []
|
||||
|
||||
// Header:
|
||||
// Nr;Name;Vorname;Straße, Hausnr.;PLZ;Wohnort;Geburtsdatum;Mobilfunknummer;Private Mail-Adresse;Kreditinstitut;IBAN;BIC;Datum;Mitgliedsstatus
|
||||
const rows: CsvMemberRow[] = []
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const cols = lines[i].split(";").map((v) => v.trim())
|
||||
if (cols.length < 14) continue
|
||||
|
||||
const number = cols[0]
|
||||
const lastname = cols[1]
|
||||
const firstname = cols[2]
|
||||
if (!number || !lastname || !firstname) continue
|
||||
|
||||
rows.push({
|
||||
number,
|
||||
lastname,
|
||||
firstname,
|
||||
street: cols[3] || "",
|
||||
zip: cols[4] || "",
|
||||
city: cols[5] || "",
|
||||
birthdate: cols[6] || "",
|
||||
mobile: cols[7] || "",
|
||||
email: cols[8] || "",
|
||||
bankInstitute: cols[9] || "",
|
||||
iban: cols[10] || "",
|
||||
bic: cols[11] || "",
|
||||
date: cols[12] || "",
|
||||
memberStatus: cols[13] || "",
|
||||
})
|
||||
}
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
async function loadBankAccountByIban(tenantId: number) {
|
||||
const rows = await db
|
||||
.select({
|
||||
id: entitybankaccounts.id,
|
||||
ibanEncrypted: entitybankaccounts.ibanEncrypted,
|
||||
})
|
||||
.from(entitybankaccounts)
|
||||
.where(eq(entitybankaccounts.tenant, tenantId))
|
||||
|
||||
const map = new Map<string, number>()
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const iban = normalizeIban(decrypt(row.ibanEncrypted as any))
|
||||
if (iban) map.set(iban, Number(row.id))
|
||||
} catch {
|
||||
// skip broken ciphertext rows
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!secrets.ENCRYPTION_KEY && process.env.ENCRYPTION_KEY) {
|
||||
secrets.ENCRYPTION_KEY = process.env.ENCRYPTION_KEY
|
||||
}
|
||||
|
||||
if (!secrets.ENCRYPTION_KEY && process.env.INFISICAL_CLIENT_ID && process.env.INFISICAL_CLIENT_SECRET) {
|
||||
await loadSecrets()
|
||||
}
|
||||
|
||||
if (!secrets.ENCRYPTION_KEY) {
|
||||
throw new Error("ENCRYPTION_KEY fehlt. Bitte ENCRYPTION_KEY setzen oder Infisical-Zugang (INFISICAL_CLIENT_ID/INFISICAL_CLIENT_SECRET) bereitstellen.")
|
||||
}
|
||||
|
||||
const absoluteCsvPath = path.resolve(csvPath)
|
||||
if (!fs.existsSync(absoluteCsvPath)) {
|
||||
throw new Error(`CSV nicht gefunden: ${absoluteCsvPath}`)
|
||||
}
|
||||
|
||||
const raw = fs.readFileSync(absoluteCsvPath, "utf8")
|
||||
const csvRows = parseCsv(raw)
|
||||
if (!csvRows.length) {
|
||||
throw new Error("Keine importierbaren Zeilen gefunden.")
|
||||
}
|
||||
|
||||
const existingMembers = await db
|
||||
.select()
|
||||
.from(customers)
|
||||
.where(and(eq(customers.tenant, TENANT_ID), eq(customers.type, "Mitglied")))
|
||||
|
||||
const memberByNumber = new Map(existingMembers.map((m) => [String(m.customerNumber), m]))
|
||||
const bankAccountByIban = await loadBankAccountByIban(TENANT_ID)
|
||||
|
||||
let createdMembers = 0
|
||||
let updatedMembers = 0
|
||||
let createdBankAccounts = 0
|
||||
let skippedNoIban = 0
|
||||
|
||||
for (const row of csvRows) {
|
||||
const iban = normalizeIban(row.iban)
|
||||
if (!iban) {
|
||||
skippedNoIban += 1
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
const fullName = `${row.firstname} ${row.lastname}`.trim()
|
||||
const birthdate = parseGermanDate(row.birthdate)
|
||||
const sepaSignedAt = parseGermanDate(row.date)
|
||||
const active = parseBoolFromStatus(row.memberStatus)
|
||||
|
||||
let bankAccountId = bankAccountByIban.get(iban) || null
|
||||
|
||||
if (!bankAccountId) {
|
||||
if (!dryRun) {
|
||||
const [created] = await db
|
||||
.insert(entitybankaccounts)
|
||||
.values({
|
||||
tenant: TENANT_ID,
|
||||
ibanEncrypted: encrypt(iban),
|
||||
bicEncrypted: encrypt(row.bic || "UNBEKANNT"),
|
||||
bankNameEncrypted: encrypt(row.bankInstitute || "Unbekannt"),
|
||||
description: "Import Mitglieder Uebersicht 2026_1",
|
||||
})
|
||||
.returning({ id: entitybankaccounts.id })
|
||||
bankAccountId = created?.id || null
|
||||
} else {
|
||||
bankAccountId = -1
|
||||
}
|
||||
if (bankAccountId) {
|
||||
bankAccountByIban.set(iban, bankAccountId)
|
||||
createdBankAccounts += 1
|
||||
}
|
||||
}
|
||||
|
||||
const existing = memberByNumber.get(String(row.number))
|
||||
const existingInfo = (existing?.infoData && typeof existing.infoData === "object")
|
||||
? { ...(existing.infoData as Record<string, any>) }
|
||||
: {}
|
||||
|
||||
const existingIds = Array.isArray(existingInfo.bankAccountIds) ? existingInfo.bankAccountIds : []
|
||||
const mergedBankAccountIds = bankAccountId && !existingIds.includes(bankAccountId)
|
||||
? [...existingIds, bankAccountId]
|
||||
: existingIds
|
||||
|
||||
const infoData = {
|
||||
...existingInfo,
|
||||
street: row.street || existingInfo.street || "",
|
||||
zip: row.zip || existingInfo.zip || "",
|
||||
city: row.city || existingInfo.city || "",
|
||||
phone: row.mobile || existingInfo.phone || "",
|
||||
email: row.email || existingInfo.email || "",
|
||||
birthdate: birthdate || existingInfo.birthdate || null,
|
||||
hasSEPA: Boolean(sepaSignedAt || existingInfo.sepaSignedAt || existingInfo.hasSEPA),
|
||||
sepaSignedAt: sepaSignedAt || existingInfo.sepaSignedAt || null,
|
||||
bankAccountIds: mergedBankAccountIds,
|
||||
}
|
||||
|
||||
const payload = {
|
||||
tenant: TENANT_ID,
|
||||
customerNumber: String(row.number),
|
||||
type: "Mitglied",
|
||||
isCompany: false,
|
||||
firstname: row.firstname,
|
||||
lastname: row.lastname,
|
||||
name: fullName,
|
||||
active,
|
||||
infoData,
|
||||
archived: false,
|
||||
}
|
||||
|
||||
if (!existing) {
|
||||
if (!dryRun) {
|
||||
const [created] = await db.insert(customers).values(payload).returning()
|
||||
if (created) memberByNumber.set(String(row.number), created)
|
||||
}
|
||||
createdMembers += 1
|
||||
} else {
|
||||
if (!dryRun) {
|
||||
await db
|
||||
.update(customers)
|
||||
.set({
|
||||
...payload,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(customers.id, existing.id), eq(customers.tenant, TENANT_ID)))
|
||||
}
|
||||
updatedMembers += 1
|
||||
}
|
||||
}
|
||||
|
||||
console.log("")
|
||||
console.log(`[IMPORT MEMBERS] Tenant: ${TENANT_ID}`)
|
||||
console.log(`[IMPORT MEMBERS] CSV: ${absoluteCsvPath}`)
|
||||
console.log(`[IMPORT MEMBERS] Dry-Run: ${dryRun ? "JA" : "NEIN"}`)
|
||||
console.log(`[IMPORT MEMBERS] Zeilen: ${csvRows.length}`)
|
||||
console.log(`[IMPORT MEMBERS] Mitglieder erstellt: ${createdMembers}`)
|
||||
console.log(`[IMPORT MEMBERS] Mitglieder aktualisiert: ${updatedMembers}`)
|
||||
console.log(`[IMPORT MEMBERS] Bankkonten erstellt: ${createdBankAccounts}`)
|
||||
console.log(`[IMPORT MEMBERS] Ohne IBAN übersprungen: ${skippedNoIban}`)
|
||||
console.log("")
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error("[IMPORT MEMBERS] Fehler:", err)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
await pool.end()
|
||||
})
|
||||
265
backend/scripts/import-skr42-accounts.ts
Normal file
265
backend/scripts/import-skr42-accounts.ts
Normal file
@@ -0,0 +1,265 @@
|
||||
import fs from "node:fs"
|
||||
import path from "node:path"
|
||||
import zlib from "node:zlib"
|
||||
|
||||
type ParsedAccount = {
|
||||
number: string
|
||||
label: string
|
||||
}
|
||||
|
||||
const DEFAULT_PDF_PATH = "/Users/florianfederspiel/Downloads/12901_DATEV-Kontenrahmen SKR 42 Vereine, Stiftungen, gGmbH (Bilanz).pdf"
|
||||
const ACCOUNT_CHART = "skr42"
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const dryRun = args.includes("--dry-run")
|
||||
const parseOnly = args.includes("--parse-only")
|
||||
const pdfArg = args.find((arg) => !arg.startsWith("--"))
|
||||
const pdfPath = path.resolve(pdfArg || DEFAULT_PDF_PATH)
|
||||
|
||||
function decodePdfString(raw: string) {
|
||||
let out = ""
|
||||
|
||||
for (let i = 0; i < raw.length; i += 1) {
|
||||
const ch = raw[i]
|
||||
|
||||
if (ch !== "\\") {
|
||||
out += ch
|
||||
continue
|
||||
}
|
||||
|
||||
const next = raw[i + 1]
|
||||
if (!next) break
|
||||
|
||||
if (next === "n") {
|
||||
out += "\n"
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (next === "r") {
|
||||
out += "\r"
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (next === "t") {
|
||||
out += "\t"
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (next === "b") {
|
||||
out += "\b"
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (next === "f") {
|
||||
out += "\f"
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (next === "(" || next === ")" || next === "\\") {
|
||||
out += next
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (/[0-7]/.test(next)) {
|
||||
let oct = next
|
||||
let advance = 1
|
||||
|
||||
for (let j = 2; j <= 3; j += 1) {
|
||||
const c = raw[i + j]
|
||||
if (!c || !/[0-7]/.test(c)) break
|
||||
oct += c
|
||||
advance += 1
|
||||
}
|
||||
|
||||
out += String.fromCharCode(parseInt(oct, 8))
|
||||
i += advance
|
||||
continue
|
||||
}
|
||||
|
||||
out += next
|
||||
i += 1
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
function extractTextFromTjOperator(segment: string) {
|
||||
const parts = segment.match(/\((?:\\.|[^\\)])*\)/g)
|
||||
if (!parts) return ""
|
||||
|
||||
return parts
|
||||
.map((p) => decodePdfString(p.slice(1, -1)))
|
||||
.join("")
|
||||
}
|
||||
|
||||
function extractPdfTextStreams(pdfBuffer: Buffer) {
|
||||
const pdfLatin = pdfBuffer.toString("latin1")
|
||||
const texts: string[] = []
|
||||
|
||||
let cursor = 0
|
||||
while (true) {
|
||||
const streamPos = pdfLatin.indexOf("stream", cursor)
|
||||
if (streamPos < 0) break
|
||||
|
||||
let dataStart = streamPos + 6
|
||||
if (pdfLatin[dataStart] === "\r" && pdfLatin[dataStart + 1] === "\n") {
|
||||
dataStart += 2
|
||||
} else if (pdfLatin[dataStart] === "\n") {
|
||||
dataStart += 1
|
||||
}
|
||||
|
||||
const streamEnd = pdfLatin.indexOf("endstream", dataStart)
|
||||
if (streamEnd < 0) break
|
||||
|
||||
const sliceEnd = streamEnd > dataStart && pdfBuffer[streamEnd - 1] === 0x0d
|
||||
? streamEnd - 1
|
||||
: streamEnd
|
||||
|
||||
const compressed = pdfBuffer.subarray(dataStart, sliceEnd)
|
||||
|
||||
try {
|
||||
const inflated = zlib.inflateSync(compressed).toString("latin1")
|
||||
texts.push(inflated)
|
||||
} catch {
|
||||
// ignore non-flate streams
|
||||
}
|
||||
|
||||
cursor = streamEnd + 9
|
||||
}
|
||||
|
||||
return texts
|
||||
}
|
||||
|
||||
function normalizeLabel(value: string) {
|
||||
return value
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/\s+-\s+/g, "-")
|
||||
.trim()
|
||||
}
|
||||
|
||||
function looksLikeAccountLabel(value: string) {
|
||||
const letters = (value.match(/[A-Za-zÄÖÜäöüß]/g) || []).length
|
||||
return letters >= 3
|
||||
}
|
||||
|
||||
function parseAccountsFromPdf(pdfBuffer: Buffer): ParsedAccount[] {
|
||||
const streams = extractPdfTextStreams(pdfBuffer)
|
||||
const found = new Map<string, string>()
|
||||
|
||||
const accountPattern = /^\s*([A-Z])?\s*(\d{3,5})\s+0\s+(.+)$/
|
||||
|
||||
for (const stream of streams) {
|
||||
const operators = stream.match(/\[(?:.|\r|\n)*?\]TJ|\((?:\\.|[^\\)])*\)Tj/g)
|
||||
if (!operators) continue
|
||||
|
||||
for (const op of operators) {
|
||||
const text = normalizeLabel(extractTextFromTjOperator(op))
|
||||
if (!text) continue
|
||||
|
||||
const m = text.match(accountPattern)
|
||||
if (m) {
|
||||
const number = m[2]
|
||||
const label = normalizeLabel(m[3])
|
||||
if (!looksLikeAccountLabel(label)) continue
|
||||
|
||||
const existing = found.get(number)
|
||||
if (!existing || label.length > existing.length) {
|
||||
found.set(number, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...found.entries()]
|
||||
.map(([number, label]) => ({ number, label }))
|
||||
.sort((a, b) => Number(a.number) - Number(b.number))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!fs.existsSync(pdfPath)) {
|
||||
throw new Error(`PDF nicht gefunden: ${pdfPath}`)
|
||||
}
|
||||
|
||||
const pdfBuffer = fs.readFileSync(pdfPath)
|
||||
const parsed = parseAccountsFromPdf(pdfBuffer)
|
||||
|
||||
if (!parsed.length) {
|
||||
throw new Error("Keine Konten aus PDF extrahiert.")
|
||||
}
|
||||
|
||||
if (parseOnly) {
|
||||
console.log("")
|
||||
console.log(`[SKR42 IMPORT] PDF: ${pdfPath}`)
|
||||
console.log(`[SKR42 IMPORT] Gefundene Konten: ${parsed.length}`)
|
||||
console.log(`[SKR42 IMPORT] Parse-Only: JA`)
|
||||
console.log("")
|
||||
console.log("[SKR42 IMPORT] Beispiel (erste 15):")
|
||||
for (const item of parsed.slice(0, 15)) {
|
||||
console.log(` ${item.number} ${item.label}`)
|
||||
}
|
||||
console.log("")
|
||||
return
|
||||
}
|
||||
|
||||
const { eq } = await import("drizzle-orm")
|
||||
const { db, pool } = await import("../db")
|
||||
const { accounts } = await import("../db/schema")
|
||||
|
||||
const existing = await db
|
||||
.select({ number: accounts.number })
|
||||
.from(accounts)
|
||||
.where(eq(accounts.accountChart, ACCOUNT_CHART))
|
||||
|
||||
const existingSet = new Set(existing.map((r) => String(r.number)))
|
||||
|
||||
const toInsert = parsed
|
||||
.filter((a) => !existingSet.has(a.number))
|
||||
.map((a) => ({
|
||||
number: a.number,
|
||||
label: a.label,
|
||||
accountChart: ACCOUNT_CHART,
|
||||
description: "DATEV SKR42 Import",
|
||||
}))
|
||||
|
||||
if (!dryRun && toInsert.length > 0) {
|
||||
const batchSize = 500
|
||||
for (let i = 0; i < toInsert.length; i += batchSize) {
|
||||
const batch = toInsert.slice(i, i + batchSize)
|
||||
await db.insert(accounts).values(batch)
|
||||
}
|
||||
}
|
||||
|
||||
console.log("")
|
||||
console.log(`[SKR42 IMPORT] PDF: ${pdfPath}`)
|
||||
console.log(`[SKR42 IMPORT] Gefundene Konten: ${parsed.length}`)
|
||||
console.log(`[SKR42 IMPORT] Bereits vorhanden (skr42): ${existing.length}`)
|
||||
console.log(`[SKR42 IMPORT] Neu einzufuegen: ${toInsert.length}`)
|
||||
console.log(`[SKR42 IMPORT] Dry-Run: ${dryRun ? "JA" : "NEIN"}`)
|
||||
console.log("")
|
||||
|
||||
if (parsed.length > 0) {
|
||||
console.log("[SKR42 IMPORT] Beispiel (erste 15):")
|
||||
for (const item of parsed.slice(0, 15)) {
|
||||
console.log(` ${item.number} ${item.label}`)
|
||||
}
|
||||
console.log("")
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error("[SKR42 IMPORT] Fehler:", err)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
if (!parseOnly) {
|
||||
const { pool } = await import("../db")
|
||||
await pool.end()
|
||||
}
|
||||
})
|
||||
BIN
backend/scripts/skr42.pdf
Normal file
BIN
backend/scripts/skr42.pdf
Normal file
Binary file not shown.
Reference in New Issue
Block a user