Merge pull request 'fix(release): stream large migration checksums from git' (#225) from flfeders/codex/release-migration-checksums into main

Reviewed-on: #225

docs/releases.md

@@ -41,7 +41,9 @@ Die Pipeline nutzt die fest gepinnten Pakete `@semantic-release/commit-analyzer`
 
 Es zählt der höchste erforderliche Sprung seit dem letzten stabilen Tag `vMAJOR.MINOR.PATCH`. Eine Änderung und ihr vollständiger Git-Revert innerhalb desselben Release-Bereichs heben sich auf. Selbst verfasste Nachrichten müssen dem Schema entsprechen; echte Merge-Commits und automatisch generierte Git-Reverts werden gesondert behandelt.
 
-Der erste Release heißt `3.0.0`. `bootstrapCommit` in `release.config.json` begrenzt die Prüfung und die erste Änderungsliste auf die Einführung dieser Versionierung. Ältere Commits werden weder umgeschrieben noch nachträglich klassifiziert. Den Ausgangscommit nach Einführung nicht ändern.
+Der erste Release heißt `3.0.0`. `bootstrapCommit` in `release.config.json` begrenzt die Prüfung und die erste Änderungsliste auf die Einführung dieser Versionierung auf `dev`. `legacyHistoryHeads` hält zusätzlich den letzten alten `main`-Stand fest (`e7554fa2…` vor der ersten Übernahme von `dev`). Dessen Historie enthält ältere Änderungen an Issue-Vorlagen, die nicht vom `dev`-Ausgangscommit erreichbar sind. Beide festgelegten Historien werden aus Commit-Prüfung, Versionsberechnung und Release Notes ausgeschlossen. Neue Commits zählen auch dann, wenn ihr Zweig von einem dieser alten Stände abzweigt.
+
+Ältere Commits werden weder umgeschrieben noch nachträglich klassifiziert. Diese festen Ausgangscommits nach Einführung nicht verschieben und keine späteren Änderungen hinzufügen, um die Commit-Prüfung zu umgehen. Der vollständige Checkout muss auch die alten Branch-Historien enthalten; ein fehlender Ausgangscommit führt zum Abbruch.
 
 ## Versionen und Artefakte
 

release.config.json

@@ -1,6 +1,7 @@
 {
   "initialVersion": "3.0.0",
   "bootstrapCommit": "cfef07b96d97f8dffdfc32c74f8f32b62fe43b18",
+  "legacyHistoryHeads": ["e7554fa2cc649f69dce4880abe76c6fab0f82700"],
   "branch": "main",
   "repositoryUrl": "https://git.federspiel.tech/flfeders/FEDEO",
   "registry": "git.federspiel.tech/flfeders/fedeo",

scripts/release/cli.mjs

@@ -5,7 +5,7 @@ try {
   if (command === 'check') {
     const config = readConfig();
     assertBootstrap(config, 'HEAD', process.cwd());
-    const commits = commitsSince(config.bootstrapCommit);
+    const commits = commitsSince(config.bootstrapCommit, 'HEAD', process.cwd(), config.legacyHistoryHeads);
     checkCommits(commits);
     console.log(`${commits.length} Commit-Nachrichten geprüft.`);
   } else if (command === 'preview') {

scripts/release/lib.mjs

@@ -1,4 +1,4 @@
-import { execFileSync } from 'node:child_process';
+import { execFileSync, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
@@ -24,9 +24,9 @@ export function validCommit(message) {
     (/^Revert ".+"$/.test(message.split('\n')[0]) && /^This reverts commit [a-f0-9]{40}\.$/m.test(message));
 }
 
-export function commitsSince(from, to = 'HEAD', cwd = process.cwd()) {
+export function commitsSince(from, to = 'HEAD', cwd = process.cwd(), excludedHeads = []) {
   // semantic-release's revert filter expects newest commits first.
-  const hashes = git(['rev-list', '--no-merges', `${from}..${to}`], cwd);
+  const hashes = git(['rev-list', '--no-merges', `${from}..${to}`, ...excludedHeads.map(head => `^${head}`)], cwd);
   return hashes ? hashes.split('\n').map(hash => ({ hash, message: git(['show', '-s', '--format=%B', hash], cwd) })) : [];
 }
 
@@ -42,6 +42,11 @@ export async function releaseType(commits, cwd = process.cwd()) {
 export function assertBootstrap(config, target, cwd) {
   try { git(['merge-base', '--is-ancestor', config.bootstrapCommit, target], cwd); }
   catch { throw new Error('Der Release-Ausgangscommit fehlt in der Historie. Vollständig auschecken (fetch-depth: 0) und dev nach main übernehmen.'); }
+  for (const head of config.legacyHistoryHeads ?? []) {
+    if (!/^[a-f0-9]{40}$/.test(head)) throw new Error('legacyHistoryHeads muss feste vollständige Commit-IDs enthalten.');
+    try { git(['rev-parse', '--verify', `${head}^{commit}`], cwd); }
+    catch { throw new Error(`Die alte Release-Historie ${head} fehlt. Alle Branches vollständig laden (fetch-depth: 0).`); }
+  }
 }
 
 export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd), target = 'HEAD' } = {}) {
@@ -53,8 +58,11 @@ export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd)
   const resume = !!lastTag && git(['rev-parse', `${lastTag}^{commit}`], cwd) === sha;
   const previousTag = resume ? tags[1] : lastTag;
   const from = previousTag ? git(['rev-parse', `${previousTag}^{commit}`], cwd) : config.bootstrapCommit;
-  const commits = commitsSince(from, sha, cwd);
-  checkCommits(commitsSince(config.bootstrapCommit, sha, cwd));
+  // Old main-only commits are not ancestors of the dev bootstrap. Exclude their
+  // pinned history without dropping new work branched from those old commits.
+  const legacyHeads = config.legacyHistoryHeads ?? [];
+  const commits = commitsSince(from, sha, cwd, legacyHeads);
+  checkCommits(commitsSince(config.bootstrapCommit, sha, cwd, legacyHeads));
   const bump = await releaseType(commits, cwd);
   const version = resume ? lastTag.slice(1) : !lastTag ? config.initialVersion : bump ? semver.inc(lastTag.slice(1), bump) : null;
   if (version && !semver.valid(version)) throw new Error('Ungültige Release-Version.');
@@ -67,7 +75,7 @@ export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd)
     lastRelease: previousTag ? { version: previousTag.slice(1), gitTag: previousTag, gitHead: from } : {},
     nextRelease: { version, gitTag: tag, gitHead: sha },
   }) : '';
-  const count = git(['rev-list', '--count', `${config.bootstrapCommit}..${sha}`], cwd);
+  const count = git(['rev-list', '--count', `${config.bootstrapCommit}..${sha}`, ...legacyHeads.map(head => `^${head}`)], cwd);
   return {
     sha, version, tag, previousTag: previousTag || null, from, bump, resume,
     developmentVersion: `${lastTag ? lastTag.slice(1) : '0.0.0'}-dev.${count}.g${sha.slice(0, 12)}`,
@@ -83,11 +91,33 @@ export function componentVersions(sha, cwd = process.cwd()) {
   };
 }
 
-export function migrationManifest(sha, cwd = process.cwd()) {
-  return ['backend/db/migrations', 'push-server/packages/db/drizzle'].flatMap(folder => {
-    const files = git(['ls-tree', '-r', '--name-only', sha, '--', folder], cwd);
-    return files ? files.split('\n').filter(path => path.endsWith('.sql')).map(path => ({
-      path, sha256: createHash('sha256').update(execFileSync('git', ['show', `${sha}:${path}`], { cwd })).digest('hex'),
-    })) : [];
+function gitBlobSha256(reference, cwd) {
+  return new Promise((resolve, reject) => {
+    const child = spawn('git', ['cat-file', 'blob', reference], { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
+    const hash = createHash('sha256');
+    let stderr = '';
+    const fail = message => reject(new Error(`Prüfsumme für ${reference} fehlgeschlagen: ${message}`));
+    // Reference-data migrations can exceed 90 MB. Hash raw chunks without
+    // buffering the entire blob or altering whitespace/encoding.
+    child.stdout.on('data', chunk => hash.update(chunk));
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(0, 8192); });
+    child.once('error', error => fail(error.message));
+    for (const stream of [child.stdout, child.stderr]) {
+      stream.once('error', error => { child.kill(); fail(error.message); });
+    }
+    child.once('close', (code, signal) => {
+      if (code !== 0) fail(`git cat-file (Exit ${code ?? signal}): ${stderr.trim()}`);
+      else resolve(hash.digest('hex'));
+    });
   });
 }
+
+export async function migrationManifest(sha, cwd = process.cwd()) {
+  const files = git(['ls-tree', '-r', '-z', '--name-only', sha, '--',
+    'backend/db/migrations', 'push-server/packages/db/drizzle'], cwd)
+    .split('\0').filter(path => path.endsWith('.sql'));
+  const manifest = [];
+  for (const path of files) manifest.push({ path, sha256: await gitBlobSha256(`${sha}:${path}`, cwd) });
+  return manifest;
+}

scripts/release/publish.mjs

@@ -14,11 +14,11 @@ export function assertManifest(manifest, plan, config) {
   }
 }
 
-export function makeManifest(plan, images, config) {
+export async function makeManifest(plan, images, config) {
   return {
     schemaVersion: 1, version: plan.version, tag: plan.tag, commit: plan.sha,
     previousTag: plan.previousTag, repository: config.repositoryUrl,
-    images, components: componentVersions(plan.sha), migrations: migrationManifest(plan.sha),
+    images, components: componentVersions(plan.sha), migrations: await migrationManifest(plan.sha),
     migrationPolicy: 'Forward migrations on startup. Downgrade requires schema compatibility or a matching database backup.',
   };
 }
@@ -61,7 +61,7 @@ export async function publishRelease({ plan, config, api, build, promote, files
   }
   if (!manifest) {
     const images = await build(plan.version);
-    manifest = manifestFactory(plan, images, config);
+    manifest = await manifestFactory(plan, images, config);
   }
   assertManifest(manifest, plan, config);
   if (!release) {

scripts/release/tests/migrations.test.mjs

@@ -0,0 +1,73 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { git, migrationManifest } from '../lib.mjs';
+
+function fixture(t) {
+  const cwd = mkdtempSync(join(tmpdir(), 'fedeo-migrations-test-'));
+  t.after(() => rmSync(cwd, { recursive: true, force: true }));
+  const run = args => git(args, cwd, { stdio: ['pipe', 'pipe', 'pipe'] });
+  run(['init', '-b', 'main']);
+  run(['config', 'user.name', 'Release Test']);
+  run(['config', 'user.email', 'release@example.test']);
+  // Keep CRLF test bytes in Git regardless of the developer's autocrlf setting.
+  writeFileSync(join(cwd, '.gitattributes'), '*.sql -text\n');
+  const write = (path, data) => {
+    mkdirSync(dirname(join(cwd, path)), { recursive: true });
+    writeFileSync(join(cwd, path), data);
+  };
+  const commit = () => {
+    run(['add', '.']);
+    run(['commit', '--allow-empty', '-m', 'test: add migrations']);
+    return run(['rev-parse', 'HEAD']);
+  };
+  return { cwd, run, write, commit };
+}
+
+test('large migration hashes use the complete committed bytes beyond process buffer limits', async t => {
+  const f = fixture(t);
+  const path = 'backend/db/migrations/0058_reference_data.sql';
+  f.write(path, '');
+  const expected = createHash('sha256');
+  const chunk = Buffer.alloc(1024 * 1024, Buffer.from("  INSERT INTO reference VALUES ('Grüße');\r\n"));
+  // Exceed both Node's 1 MiB default and the text Git helper's 32 MiB buffer.
+  for (let i = 0; i < 34; i++) {
+    appendFileSync(join(f.cwd, path), chunk);
+    expected.update(chunk);
+  }
+  const sha = f.commit();
+  f.write(path, 'local changes must not affect the release checksum');
+  assert.deepEqual(await migrationManifest(sha, f.cwd), [{ path, sha256: expected.digest('hex') }]);
+});
+
+test('both migration folders are hashed byte-for-byte and non-SQL files are excluded', async t => {
+  const f = fixture(t);
+  const files = [
+    ['backend/db/migrations/0000_init.sql', Buffer.from(' \r\nSELECT 1;\r\n\t')],
+    ['push-server/packages/db/drizzle/0000_init.sql', Buffer.from('SELECT \'äöü\';\n')],
+  ];
+  for (const [path, content] of files) f.write(path, content);
+  f.write('backend/db/migrations/meta/snapshot.json', '{}');
+  const sha = f.commit();
+  assert.deepEqual(await migrationManifest(sha, f.cwd), files.map(([path, content]) => ({
+    path, sha256: createHash('sha256').update(content).digest('hex'),
+  })));
+});
+
+test('a commit without migration folders produces an empty manifest', async t => {
+  const f = fixture(t);
+  assert.deepEqual(await migrationManifest(f.commit(), f.cwd), []);
+});
+
+test('an unreadable Git blob rejects the manifest instead of recording a partial checksum', async t => {
+  const f = fixture(t);
+  const path = 'backend/db/migrations/0000_missing.sql';
+  f.write(path, 'SELECT 1;');
+  const sha = f.commit();
+  const blob = f.run(['rev-parse', `${sha}:${path}`]);
+  rmSync(join(f.cwd, '.git', 'objects', blob.slice(0, 2), blob.slice(2)));
+  await assert.rejects(async () => migrationManifest(sha, f.cwd), /0000_missing\.sql/);
+});

scripts/release/tests/planning.test.mjs

@@ -1,5 +1,6 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
 import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
@@ -17,9 +18,9 @@ function fixture(t) {
   run(['commit', '--allow-empty', '-m', 'historic baseline']);
   const config = { bootstrapCommit: run(['rev-parse', 'HEAD']), initialVersion: '1.0.0', repositoryUrl: 'https://git.example.test/owner/fedeo' };
   let count = 0;
-  const commit = message => {
-    writeFileSync(join(cwd, 'file'), `${++count}\n`);
-    run(['add', 'file']);
+  const commit = (message, file = 'file') => {
+    writeFileSync(join(cwd, file), `${++count}\n`);
+    run(['add', file]);
     run(['commit', '-m', message]);
     return run(['rev-parse', 'HEAD']);
   };
@@ -52,6 +53,51 @@ test('first release uses explicit baseline and excludes legacy history', async t
   assert.doesNotMatch(plan.notes, /historic baseline|compare\/\.\.\./);
 });
 
+function divergentHistory(t) {
+  const f = fixture(t);
+  f.run(['branch', 'legacy-main']);
+  f.config.bootstrapCommit = f.commit('chore: versioning baseline');
+  f.commit('feat: current export');
+  f.run(['checkout', 'legacy-main']);
+  f.commit('.gitea/ISSUE_TEMPLATE/bug_report.md aktualisiert', 'legacy-template');
+  const legacyHead = f.commit('feat: historic main feature', 'legacy-feature');
+  f.config.legacyHistoryHeads = [legacyHead];
+  return f;
+}
+
+test('the first main merge excludes pinned legacy branch history from validation and notes', async t => {
+  const f = divergentHistory(t);
+  f.run(['checkout', 'main']);
+  f.run(['merge', '--no-ff', 'legacy-main', '-m', 'Merge historic main']);
+  const plan = await f.plan();
+  assert.equal(plan.version, '1.0.0');
+  assert.match(plan.notes, /current export/);
+  assert.doesNotMatch(plan.notes, /historic main feature|ISSUE_TEMPLATE/);
+  writeFileSync(join(f.cwd, 'release.config.json'), JSON.stringify(f.config));
+  const output = execFileSync(process.execPath, [resolve('scripts/release/cli.mjs'), 'check'], { cwd: f.cwd, encoding: 'utf8' });
+  assert.match(output, /1 Commit-Nachrichten geprüft/);
+});
+
+test('new commits on a legacy branch still affect versions and must use Conventional Commits', async t => {
+  const f = divergentHistory(t);
+  f.run(['checkout', 'main']);
+  f.run(['tag', 'v1.0.0']);
+  f.run(['checkout', 'legacy-main']);
+  f.commit('feat(api)!: replace exported API', 'new-api');
+  f.run(['checkout', 'main']);
+  f.run(['merge', '--no-ff', 'legacy-main', '-m', 'Merge new API']);
+  const plan = await f.plan();
+  assert.equal(plan.version, '2.0.0');
+  assert.match(plan.notes, /replace exported API/);
+  assert.doesNotMatch(plan.notes, /historic main feature/);
+
+  f.run(['checkout', 'legacy-main']);
+  f.commit('Unclassified new change', 'another-change');
+  f.run(['checkout', 'main']);
+  f.run(['merge', '--no-ff', 'legacy-main', '-m', 'Merge another change']);
+  await assert.rejects(f.plan(), /Conventional Commits fehlen:[\s\S]*Unclassified new change/);
+});
+
 test('patch/minor/major releases, multi-digit tags and unrelated component tags', async t => {
   const f = fixture(t);
   f.run(['tag', 'v1.9.0']);
@@ -104,6 +150,14 @@ test('an unknown baseline fails instead of releasing unrelated history', async t
   await assert.rejects(f.plan(), /Ausgangscommit/);
 });
 
+test('missing or mutable legacy history heads fail instead of silently skipping validation', async t => {
+  const f = fixture(t);
+  f.config.legacyHistoryHeads = ['f'.repeat(40)];
+  await assert.rejects(f.plan(), /alte Release-Historie .* fehlt/);
+  f.config.legacyHistoryHeads = ['main'];
+  await assert.rejects(f.plan(), /feste vollständige Commit-IDs/);
+});
+
 test('Git lease lock excludes concurrent runners and cannot remove another owner', async t => {
   const f = fixture(t);
   const remote = join(f.cwd, 'remote.git');

scripts/release/tests/publication.test.mjs

@@ -58,6 +58,26 @@ test('failed build never creates a draft or changes release channels', async ()
   assert.deepEqual(f.events, []);
 });
 
+test('asynchronous manifest generation finishes before creating the release draft', async () => {
+  const f = fixture();
+  f.dependencies.manifestFactory = async () => {
+    await new Promise(resolve => setImmediate(resolve));
+    f.events.push('manifest');
+    return f.manifest;
+  };
+  await publishRelease(f.dependencies);
+  assert.deepEqual(f.events.slice(0, 3), ['build', 'manifest', 'draft']);
+  assert.equal(f.getRelease().draft, false);
+});
+
+test('failed migration hashing never creates a draft or promotes images', async () => {
+  const f = fixture();
+  f.dependencies.manifestFactory = async () => { throw new Error('migration hashing failed'); };
+  await assert.rejects(publishRelease(f.dependencies), /migration hashing failed/);
+  assert.deepEqual(f.events, ['build']);
+  assert.equal(f.getRelease(), null);
+});
+
 test('retry a draft uses saved digests and original notes without rebuilding', async () => {
   const f = fixture();
   f.dependencies.promote = async () => { throw new Error('registry interrupted'); };