Diagnose tenant lock migration blockers

backend/scripts/apply-tenant-export-lock-migration.ts

@@ -12,6 +12,65 @@ const logStep = (message: string) => {
     console.log(`[tenant-lock-migration] ${message}`)
 }
 
+const printRelationLocks = async (db: ReturnType<typeof drizzle>, relationName: string) => {
+    const result = await db.execute(sql`
+        SELECT
+            activity.pid,
+            activity.state,
+            activity.wait_event_type,
+            activity.wait_event,
+            now() - activity.query_start AS query_age,
+            locks.mode,
+            locks.granted,
+            left(activity.query, 500) AS query
+        FROM pg_locks locks
+        JOIN pg_class relation ON relation.oid = locks.relation
+        LEFT JOIN pg_stat_activity activity ON activity.pid = locks.pid
+        WHERE relation.relname = ${relationName}
+        ORDER BY locks.granted ASC, activity.query_start ASC NULLS LAST;
+    `)
+
+    if (!result.rows.length) {
+        logStep(`Keine aktiven Locks fuer ${relationName} gefunden`)
+        return
+    }
+
+    logStep(`Aktive Locks fuer ${relationName}:`)
+    for (const row of result.rows as Record<string, any>[]) {
+        console.log(JSON.stringify({
+            pid: row.pid,
+            state: row.state,
+            waitEventType: row.wait_event_type,
+            waitEvent: row.wait_event,
+            queryAge: row.query_age,
+            mode: row.mode,
+            granted: row.granted,
+            query: row.query,
+        }, null, 2))
+    }
+}
+
+const executeStep = async (
+    db: ReturnType<typeof drizzle>,
+    message: string,
+    statement: ReturnType<typeof sql>,
+    lockRelationName?: string,
+) => {
+    logStep(message)
+
+    try {
+        await db.execute(statement)
+    } catch (err: any) {
+        const code = err?.cause?.code || err?.code
+        if (lockRelationName && ["55P03", "57014"].includes(code)) {
+            logStep(`Konnte Lock fuer ${lockRelationName} nicht rechtzeitig bekommen`)
+            await printRelationLocks(db, lockRelationName)
+        }
+
+        throw err
+    }
+}
+
 async function run() {
     let connectionString = process.env.DATABASE_URL
 
@@ -42,24 +101,22 @@ async function run() {
         logStep("Teste DB-Verbindung")
         await db.execute(sql`SELECT 1`)
 
-        logStep("Setze Statement-Timeout")
-        await db.execute(sql`SET statement_timeout = '30s'`)
+        logStep("Setze Lock- und Statement-Timeout")
+        await db.execute(sql`SET lock_timeout = '10s'`)
+        await db.execute(sql`SET statement_timeout = '2min'`)
 
-        logStep("Erweitere tenant_export_jobs")
-        await db.execute(sql`
+        await executeStep(db, "Erweitere tenant_export_jobs", sql`
             ALTER TABLE "tenant_export_jobs"
                 ADD COLUMN IF NOT EXISTS "operation" text DEFAULT 'export' NOT NULL,
                 ADD COLUMN IF NOT EXISTS "previous_tenant_locked" "locked_tenant";
-        `)
+        `, "tenant_export_jobs")
 
-        logStep("Erweitere tenants")
-        await db.execute(sql`
+        await executeStep(db, "Erweitere tenants", sql`
             ALTER TABLE "tenants"
                 ADD COLUMN IF NOT EXISTS "locked_by_export_job_id" uuid;
-        `)
+        `, "tenants")
 
-        logStep("Prüfe Foreign Key")
-        await db.execute(sql`
+        await executeStep(db, "Pruefe Foreign Key", sql`
             DO $$
             BEGIN
                 IF NOT EXISTS (
@@ -75,15 +132,13 @@ async function run() {
                         ON DELETE SET NULL;
                 END IF;
             END $$;
-        `)
+        `, "tenants")
 
-        logStep("Prüfe Drizzle-Schema")
-        await db.execute(sql`
+        await executeStep(db, "Pruefe Drizzle-Schema", sql`
             CREATE SCHEMA IF NOT EXISTS drizzle;
         `)
 
-        logStep("Prüfe Drizzle-Migrationstabelle")
-        await db.execute(sql`
+        await executeStep(db, "Pruefe Drizzle-Migrationstabelle", sql`
             CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
                 id SERIAL PRIMARY KEY,
                 hash text NOT NULL,
@@ -91,8 +146,7 @@ async function run() {
             );
         `)
 
-        logStep("Setze Drizzle-Journal auf 0055")
-        await db.execute(sql`
+        await executeStep(db, "Setze Drizzle-Journal auf 0055", sql`
             INSERT INTO drizzle.__drizzle_migrations (hash, created_at)
             SELECT 'manual-0055_tenant_export_maintenance_lock', 1784021669805
             WHERE NOT EXISTS (