From 74c0cf40b3e3404aac840c9197035b7a30107650 Mon Sep 17 00:00:00 2001 From: flfeders Date: Wed, 23 Sep 2026 15:37:19 +0200 Subject: [PATCH] fix(release): stream large migration checksums from git --- scripts/release/lib.mjs | 36 ++++++++--- scripts/release/publish.mjs | 6 +- scripts/release/tests/migrations.test.mjs | 73 ++++++++++++++++++++++ scripts/release/tests/publication.test.mjs | 20 ++++++ 4 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 scripts/release/tests/migrations.test.mjs diff --git a/scripts/release/lib.mjs b/scripts/release/lib.mjs index 202bdc6b..bbca9f95 100644 --- a/scripts/release/lib.mjs +++ b/scripts/release/lib.mjs @@ -1,4 +1,4 @@ -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; @@ -91,11 +91,33 @@ export function componentVersions(sha, cwd = process.cwd()) { }; } -export function migrationManifest(sha, cwd = process.cwd()) { - return ['backend/db/migrations', 'push-server/packages/db/drizzle'].flatMap(folder => { - const files = git(['ls-tree', '-r', '--name-only', sha, '--', folder], cwd); - return files ? files.split('\n').filter(path => path.endsWith('.sql')).map(path => ({ - path, sha256: createHash('sha256').update(execFileSync('git', ['show', `${sha}:${path}`], { cwd })).digest('hex'), - })) : []; +function gitBlobSha256(reference, cwd) { + return new Promise((resolve, reject) => { + const child = spawn('git', ['cat-file', 'blob', reference], { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + const hash = createHash('sha256'); + let stderr = ''; + const fail = message => reject(new Error(`Prüfsumme für ${reference} fehlgeschlagen: ${message}`)); + // Reference-data migrations can exceed 90 MB. Hash raw chunks without + // buffering the entire blob or altering whitespace/encoding. + child.stdout.on('data', chunk => hash.update(chunk)); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(0, 8192); }); + child.once('error', error => fail(error.message)); + for (const stream of [child.stdout, child.stderr]) { + stream.once('error', error => { child.kill(); fail(error.message); }); + } + child.once('close', (code, signal) => { + if (code !== 0) fail(`git cat-file (Exit ${code ?? signal}): ${stderr.trim()}`); + else resolve(hash.digest('hex')); + }); }); } + +export async function migrationManifest(sha, cwd = process.cwd()) { + const files = git(['ls-tree', '-r', '-z', '--name-only', sha, '--', + 'backend/db/migrations', 'push-server/packages/db/drizzle'], cwd) + .split('\0').filter(path => path.endsWith('.sql')); + const manifest = []; + for (const path of files) manifest.push({ path, sha256: await gitBlobSha256(`${sha}:${path}`, cwd) }); + return manifest; +} diff --git a/scripts/release/publish.mjs b/scripts/release/publish.mjs index 54310f09..0c1c778f 100644 --- a/scripts/release/publish.mjs +++ b/scripts/release/publish.mjs @@ -14,11 +14,11 @@ export function assertManifest(manifest, plan, config) { } } -export function makeManifest(plan, images, config) { +export async function makeManifest(plan, images, config) { return { schemaVersion: 1, version: plan.version, tag: plan.tag, commit: plan.sha, previousTag: plan.previousTag, repository: config.repositoryUrl, - images, components: componentVersions(plan.sha), migrations: migrationManifest(plan.sha), + images, components: componentVersions(plan.sha), migrations: await migrationManifest(plan.sha), migrationPolicy: 'Forward migrations on startup. Downgrade requires schema compatibility or a matching database backup.', }; } @@ -61,7 +61,7 @@ export async function publishRelease({ plan, config, api, build, promote, files } if (!manifest) { const images = await build(plan.version); - manifest = manifestFactory(plan, images, config); + manifest = await manifestFactory(plan, images, config); } assertManifest(manifest, plan, config); if (!release) { diff --git a/scripts/release/tests/migrations.test.mjs b/scripts/release/tests/migrations.test.mjs new file mode 100644 index 00000000..4c255511 --- /dev/null +++ b/scripts/release/tests/migrations.test.mjs @@ -0,0 +1,73 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { git, migrationManifest } from '../lib.mjs'; + +function fixture(t) { + const cwd = mkdtempSync(join(tmpdir(), 'fedeo-migrations-test-')); + t.after(() => rmSync(cwd, { recursive: true, force: true })); + const run = args => git(args, cwd, { stdio: ['pipe', 'pipe', 'pipe'] }); + run(['init', '-b', 'main']); + run(['config', 'user.name', 'Release Test']); + run(['config', 'user.email', 'release@example.test']); + // Keep CRLF test bytes in Git regardless of the developer's autocrlf setting. + writeFileSync(join(cwd, '.gitattributes'), '*.sql -text\n'); + const write = (path, data) => { + mkdirSync(dirname(join(cwd, path)), { recursive: true }); + writeFileSync(join(cwd, path), data); + }; + const commit = () => { + run(['add', '.']); + run(['commit', '--allow-empty', '-m', 'test: add migrations']); + return run(['rev-parse', 'HEAD']); + }; + return { cwd, run, write, commit }; +} + +test('large migration hashes use the complete committed bytes beyond process buffer limits', async t => { + const f = fixture(t); + const path = 'backend/db/migrations/0058_reference_data.sql'; + f.write(path, ''); + const expected = createHash('sha256'); + const chunk = Buffer.alloc(1024 * 1024, Buffer.from(" INSERT INTO reference VALUES ('Grüße');\r\n")); + // Exceed both Node's 1 MiB default and the text Git helper's 32 MiB buffer. + for (let i = 0; i < 34; i++) { + appendFileSync(join(f.cwd, path), chunk); + expected.update(chunk); + } + const sha = f.commit(); + f.write(path, 'local changes must not affect the release checksum'); + assert.deepEqual(await migrationManifest(sha, f.cwd), [{ path, sha256: expected.digest('hex') }]); +}); + +test('both migration folders are hashed byte-for-byte and non-SQL files are excluded', async t => { + const f = fixture(t); + const files = [ + ['backend/db/migrations/0000_init.sql', Buffer.from(' \r\nSELECT 1;\r\n\t')], + ['push-server/packages/db/drizzle/0000_init.sql', Buffer.from('SELECT \'äöü\';\n')], + ]; + for (const [path, content] of files) f.write(path, content); + f.write('backend/db/migrations/meta/snapshot.json', '{}'); + const sha = f.commit(); + assert.deepEqual(await migrationManifest(sha, f.cwd), files.map(([path, content]) => ({ + path, sha256: createHash('sha256').update(content).digest('hex'), + }))); +}); + +test('a commit without migration folders produces an empty manifest', async t => { + const f = fixture(t); + assert.deepEqual(await migrationManifest(f.commit(), f.cwd), []); +}); + +test('an unreadable Git blob rejects the manifest instead of recording a partial checksum', async t => { + const f = fixture(t); + const path = 'backend/db/migrations/0000_missing.sql'; + f.write(path, 'SELECT 1;'); + const sha = f.commit(); + const blob = f.run(['rev-parse', `${sha}:${path}`]); + rmSync(join(f.cwd, '.git', 'objects', blob.slice(0, 2), blob.slice(2))); + await assert.rejects(async () => migrationManifest(sha, f.cwd), /0000_missing\.sql/); +}); diff --git a/scripts/release/tests/publication.test.mjs b/scripts/release/tests/publication.test.mjs index b8d9ccce..e1312497 100644 --- a/scripts/release/tests/publication.test.mjs +++ b/scripts/release/tests/publication.test.mjs @@ -58,6 +58,26 @@ test('failed build never creates a draft or changes release channels', async () assert.deepEqual(f.events, []); }); +test('asynchronous manifest generation finishes before creating the release draft', async () => { + const f = fixture(); + f.dependencies.manifestFactory = async () => { + await new Promise(resolve => setImmediate(resolve)); + f.events.push('manifest'); + return f.manifest; + }; + await publishRelease(f.dependencies); + assert.deepEqual(f.events.slice(0, 3), ['build', 'manifest', 'draft']); + assert.equal(f.getRelease().draft, false); +}); + +test('failed migration hashing never creates a draft or promotes images', async () => { + const f = fixture(); + f.dependencies.manifestFactory = async () => { throw new Error('migration hashing failed'); }; + await assert.rejects(publishRelease(f.dependencies), /migration hashing failed/); + assert.deepEqual(f.events, ['build']); + assert.equal(f.getRelease(), null); +}); + test('retry a draft uses saved digests and original notes without rebuilding', async () => { const f = fixture(); f.dependencies.promote = async () => { throw new Error('registry interrupted'); }; -- 2.49.1