diff --git a/backend/scripts/apply-tenant-export-lock-migration.ts b/backend/scripts/apply-tenant-export-lock-migration.ts index d9527ad..e362ffd 100644 --- a/backend/scripts/apply-tenant-export-lock-migration.ts +++ b/backend/scripts/apply-tenant-export-lock-migration.ts @@ -12,6 +12,65 @@ const logStep = (message: string) => { console.log(`[tenant-lock-migration] ${message}`) } +const printRelationLocks = async (db: ReturnType, relationName: string) => { + const result = await db.execute(sql` + SELECT + activity.pid, + activity.state, + activity.wait_event_type, + activity.wait_event, + now() - activity.query_start AS query_age, + locks.mode, + locks.granted, + left(activity.query, 500) AS query + FROM pg_locks locks + JOIN pg_class relation ON relation.oid = locks.relation + LEFT JOIN pg_stat_activity activity ON activity.pid = locks.pid + WHERE relation.relname = ${relationName} + ORDER BY locks.granted ASC, activity.query_start ASC NULLS LAST; + `) + + if (!result.rows.length) { + logStep(`Keine aktiven Locks fuer ${relationName} gefunden`) + return + } + + logStep(`Aktive Locks fuer ${relationName}:`) + for (const row of result.rows as Record[]) { + console.log(JSON.stringify({ + pid: row.pid, + state: row.state, + waitEventType: row.wait_event_type, + waitEvent: row.wait_event, + queryAge: row.query_age, + mode: row.mode, + granted: row.granted, + query: row.query, + }, null, 2)) + } +} + +const executeStep = async ( + db: ReturnType, + message: string, + statement: ReturnType, + lockRelationName?: string, +) => { + logStep(message) + + try { + await db.execute(statement) + } catch (err: any) { + const code = err?.cause?.code || err?.code + if (lockRelationName && ["55P03", "57014"].includes(code)) { + logStep(`Konnte Lock fuer ${lockRelationName} nicht rechtzeitig bekommen`) + await printRelationLocks(db, lockRelationName) + } + + throw err + } +} + async function run() { let connectionString = process.env.DATABASE_URL @@ -42,24 +101,22 @@ async function run() { logStep("Teste DB-Verbindung") await db.execute(sql`SELECT 1`) - logStep("Setze Statement-Timeout") - await db.execute(sql`SET statement_timeout = '30s'`) + logStep("Setze Lock- und Statement-Timeout") + await db.execute(sql`SET lock_timeout = '10s'`) + await db.execute(sql`SET statement_timeout = '2min'`) - logStep("Erweitere tenant_export_jobs") - await db.execute(sql` + await executeStep(db, "Erweitere tenant_export_jobs", sql` ALTER TABLE "tenant_export_jobs" ADD COLUMN IF NOT EXISTS "operation" text DEFAULT 'export' NOT NULL, ADD COLUMN IF NOT EXISTS "previous_tenant_locked" "locked_tenant"; - `) + `, "tenant_export_jobs") - logStep("Erweitere tenants") - await db.execute(sql` + await executeStep(db, "Erweitere tenants", sql` ALTER TABLE "tenants" ADD COLUMN IF NOT EXISTS "locked_by_export_job_id" uuid; - `) + `, "tenants") - logStep("Prüfe Foreign Key") - await db.execute(sql` + await executeStep(db, "Pruefe Foreign Key", sql` DO $$ BEGIN IF NOT EXISTS ( @@ -75,15 +132,13 @@ async function run() { ON DELETE SET NULL; END IF; END $$; - `) + `, "tenants") - logStep("Prüfe Drizzle-Schema") - await db.execute(sql` + await executeStep(db, "Pruefe Drizzle-Schema", sql` CREATE SCHEMA IF NOT EXISTS drizzle; `) - logStep("Prüfe Drizzle-Migrationstabelle") - await db.execute(sql` + await executeStep(db, "Pruefe Drizzle-Migrationstabelle", sql` CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations ( id SERIAL PRIMARY KEY, hash text NOT NULL, @@ -91,8 +146,7 @@ async function run() { ); `) - logStep("Setze Drizzle-Journal auf 0055") - await db.execute(sql` + await executeStep(db, "Setze Drizzle-Journal auf 0055", sql` INSERT INTO drizzle.__drizzle_migrations (hash, created_at) SELECT 'manual-0055_tenant_export_maintenance_lock', 1784021669805 WHERE NOT EXISTS (