Merge pull request 'fix(release): stream large migration checksums from git' (#225) from flfeders/codex/release-migration-checksums into main

Reviewed-on: #225

scripts/release/lib.mjs

@@ -1,4 +1,4 @@
-import { execFileSync } from 'node:child_process';
+import { execFileSync, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
@@ -91,11 +91,33 @@ export function componentVersions(sha, cwd = process.cwd()) {
   };
 }
 
-export function migrationManifest(sha, cwd = process.cwd()) {
+function gitBlobSha256(reference, cwd) {
-  return ['backend/db/migrations', 'push-server/packages/db/drizzle'].flatMap(folder => {
+  return new Promise((resolve, reject) => {
-    const files = git(['ls-tree', '-r', '--name-only', sha, '--', folder], cwd);
+    const child = spawn('git', ['cat-file', 'blob', reference], { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
-    return files ? files.split('\n').filter(path => path.endsWith('.sql')).map(path => ({
+    const hash = createHash('sha256');
-      path, sha256: createHash('sha256').update(execFileSync('git', ['show', `${sha}:${path}`], { cwd })).digest('hex'),
+    let stderr = '';
-    })) : [];
+    const fail = message => reject(new Error(`Prüfsumme für ${reference} fehlgeschlagen: ${message}`));
+    // Reference-data migrations can exceed 90 MB. Hash raw chunks without
+    // buffering the entire blob or altering whitespace/encoding.
+    child.stdout.on('data', chunk => hash.update(chunk));
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(0, 8192); });
+    child.once('error', error => fail(error.message));
+    for (const stream of [child.stdout, child.stderr]) {
+      stream.once('error', error => { child.kill(); fail(error.message); });
+    }
+    child.once('close', (code, signal) => {
+      if (code !== 0) fail(`git cat-file (Exit ${code ?? signal}): ${stderr.trim()}`);
+      else resolve(hash.digest('hex'));
+    });
   });
 }
+
+export async function migrationManifest(sha, cwd = process.cwd()) {
+  const files = git(['ls-tree', '-r', '-z', '--name-only', sha, '--',
+    'backend/db/migrations', 'push-server/packages/db/drizzle'], cwd)
+    .split('\0').filter(path => path.endsWith('.sql'));
+  const manifest = [];
+  for (const path of files) manifest.push({ path, sha256: await gitBlobSha256(`${sha}:${path}`, cwd) });
+  return manifest;
+}

scripts/release/publish.mjs

@@ -14,11 +14,11 @@ export function assertManifest(manifest, plan, config) {
   }
 }
 
-export function makeManifest(plan, images, config) {
+export async function makeManifest(plan, images, config) {
   return {
     schemaVersion: 1, version: plan.version, tag: plan.tag, commit: plan.sha,
     previousTag: plan.previousTag, repository: config.repositoryUrl,
-    images, components: componentVersions(plan.sha), migrations: migrationManifest(plan.sha),
+    images, components: componentVersions(plan.sha), migrations: await migrationManifest(plan.sha),
     migrationPolicy: 'Forward migrations on startup. Downgrade requires schema compatibility or a matching database backup.',
   };
 }
@@ -61,7 +61,7 @@ export async function publishRelease({ plan, config, api, build, promote, files
   }
   if (!manifest) {
     const images = await build(plan.version);
-    manifest = manifestFactory(plan, images, config);
+    manifest = await manifestFactory(plan, images, config);
   }
   assertManifest(manifest, plan, config);
   if (!release) {

scripts/release/tests/migrations.test.mjs

@@ -0,0 +1,73 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { git, migrationManifest } from '../lib.mjs';
+
+function fixture(t) {
+  const cwd = mkdtempSync(join(tmpdir(), 'fedeo-migrations-test-'));
+  t.after(() => rmSync(cwd, { recursive: true, force: true }));
+  const run = args => git(args, cwd, { stdio: ['pipe', 'pipe', 'pipe'] });
+  run(['init', '-b', 'main']);
+  run(['config', 'user.name', 'Release Test']);
+  run(['config', 'user.email', 'release@example.test']);
+  // Keep CRLF test bytes in Git regardless of the developer's autocrlf setting.
+  writeFileSync(join(cwd, '.gitattributes'), '*.sql -text\n');
+  const write = (path, data) => {
+    mkdirSync(dirname(join(cwd, path)), { recursive: true });
+    writeFileSync(join(cwd, path), data);
+  };
+  const commit = () => {
+    run(['add', '.']);
+    run(['commit', '--allow-empty', '-m', 'test: add migrations']);
+    return run(['rev-parse', 'HEAD']);
+  };
+  return { cwd, run, write, commit };
+}
+
+test('large migration hashes use the complete committed bytes beyond process buffer limits', async t => {
+  const f = fixture(t);
+  const path = 'backend/db/migrations/0058_reference_data.sql';
+  f.write(path, '');
+  const expected = createHash('sha256');
+  const chunk = Buffer.alloc(1024 * 1024, Buffer.from("  INSERT INTO reference VALUES ('Grüße');\r\n"));
+  // Exceed both Node's 1 MiB default and the text Git helper's 32 MiB buffer.
+  for (let i = 0; i < 34; i++) {
+    appendFileSync(join(f.cwd, path), chunk);
+    expected.update(chunk);
+  }
+  const sha = f.commit();
+  f.write(path, 'local changes must not affect the release checksum');
+  assert.deepEqual(await migrationManifest(sha, f.cwd), [{ path, sha256: expected.digest('hex') }]);
+});
+
+test('both migration folders are hashed byte-for-byte and non-SQL files are excluded', async t => {
+  const f = fixture(t);
+  const files = [
+    ['backend/db/migrations/0000_init.sql', Buffer.from(' \r\nSELECT 1;\r\n\t')],
+    ['push-server/packages/db/drizzle/0000_init.sql', Buffer.from('SELECT \'äöü\';\n')],
+  ];
+  for (const [path, content] of files) f.write(path, content);
+  f.write('backend/db/migrations/meta/snapshot.json', '{}');
+  const sha = f.commit();
+  assert.deepEqual(await migrationManifest(sha, f.cwd), files.map(([path, content]) => ({
+    path, sha256: createHash('sha256').update(content).digest('hex'),
+  })));
+});
+
+test('a commit without migration folders produces an empty manifest', async t => {
+  const f = fixture(t);
+  assert.deepEqual(await migrationManifest(f.commit(), f.cwd), []);
+});
+
+test('an unreadable Git blob rejects the manifest instead of recording a partial checksum', async t => {
+  const f = fixture(t);
+  const path = 'backend/db/migrations/0000_missing.sql';
+  f.write(path, 'SELECT 1;');
+  const sha = f.commit();
+  const blob = f.run(['rev-parse', `${sha}:${path}`]);
+  rmSync(join(f.cwd, '.git', 'objects', blob.slice(0, 2), blob.slice(2)));
+  await assert.rejects(async () => migrationManifest(sha, f.cwd), /0000_missing\.sql/);
+});

scripts/release/tests/publication.test.mjs

@@ -58,6 +58,26 @@ test('failed build never creates a draft or changes release channels', async ()
   assert.deepEqual(f.events, []);
 });
 
+test('asynchronous manifest generation finishes before creating the release draft', async () => {
+  const f = fixture();
+  f.dependencies.manifestFactory = async () => {
+    await new Promise(resolve => setImmediate(resolve));
+    f.events.push('manifest');
+    return f.manifest;
+  };
+  await publishRelease(f.dependencies);
+  assert.deepEqual(f.events.slice(0, 3), ['build', 'manifest', 'draft']);
+  assert.equal(f.getRelease().draft, false);
+});
+
+test('failed migration hashing never creates a draft or promotes images', async () => {
+  const f = fixture();
+  f.dependencies.manifestFactory = async () => { throw new Error('migration hashing failed'); };
+  await assert.rejects(publishRelease(f.dependencies), /migration hashing failed/);
+  assert.deepEqual(f.events, ['build']);
+  assert.equal(f.getRelease(), null);
+});
+
 test('retry a draft uses saved digests and original notes without rebuilding', async () => {
   const f = fixture();
   f.dependencies.promote = async () => { throw new Error('registry interrupted'); };