feat(storage): add verified offline MinIO migration pilot
This commit is contained in:
112
scripts/storage/tests/integration.mjs
Normal file
112
scripts/storage/tests/integration.mjs
Normal file
@@ -0,0 +1,112 @@
|
||||
// Explicit integration run; never discovers or connects to an existing FEDEO stack.
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtemp, mkdir, writeFile, readFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { client, call, contract, inventory, transfer, verifyManifest, MAX_OBJECT_BYTES } from '../s3.mjs';
|
||||
|
||||
const docker = (...args) => execFileSync('docker', args, { encoding: 'utf8', timeout: 120_000, stdio: ['ignore', 'pipe', 'pipe'] }).trim();
|
||||
const spec = JSON.parse(await readFile(new URL('../seaweedfs.json', import.meta.url), 'utf8'));
|
||||
const root = await mkdtemp(join(tmpdir(), 'fedeo-storage-contract-'));
|
||||
const prefix = `fedeo-contract-${randomUUID().slice(0, 8)}`;
|
||||
const containers = [];
|
||||
const Bucket = 'fedeo';
|
||||
const credentials = { accessKeyId: 'fedeo-test-user', secretAccessKey: 'fedeo-test-secret-only' };
|
||||
const adminCredentials = { accessKeyId: 'fedeo-test-admin', secretAccessKey: 'fedeo-test-admin-secret' };
|
||||
process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED';
|
||||
const config = endpoint => ({ endpoint, region: 'eu-central-1', credentials });
|
||||
let source, target, admin;
|
||||
function run(name, ...args) { containers.push(name); return docker('run', '-d', '--pull', 'never', '--name', name, ...args); }
|
||||
function endpoint(name) { return 'http://' + docker('port', name, '9000/tcp'); }
|
||||
async function ready(s3) {
|
||||
for (let i = 0; i < 90; i++) {
|
||||
try { await call(s3, 'ListBuckets', {}); return; }
|
||||
catch { await new Promise(resolve => setTimeout(resolve, 1000)); }
|
||||
}
|
||||
throw new Error('S3 did not become ready');
|
||||
}
|
||||
async function weed(name, data) {
|
||||
await mkdir(join(data, 'master'), { recursive: true });
|
||||
run(name, '-p', '127.0.0.1::9000', '-v', `${data}:/data`, '-v', `${root}/config:/etc/fedeo:ro`,
|
||||
'-e', 'FEDEO_S3_CONFIG_SHA256=' + createHash('sha256').update(await readFile(join(root, 'config/s3.json'))).digest('hex'),
|
||||
'--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command);
|
||||
return endpoint(name);
|
||||
}
|
||||
try {
|
||||
await mkdir(join(root, 'config'));
|
||||
await writeFile(join(root, 'config/start-seaweedfs.sh'), await readFile(new URL('../start-seaweedfs.sh', import.meta.url)));
|
||||
assert.throws(() => docker('run', '--rm', '--pull', 'never', '-v', `${root}/config:/etc/fedeo:ro`,
|
||||
'--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command));
|
||||
await writeFile(join(root, 'config/s3.json'), JSON.stringify({ identities: [
|
||||
{ name: 'admin', credentials: [adminCredentials], actions: ['Admin', 'Read', 'Write', 'List', 'Tagging'] },
|
||||
{ name: 'fedeo', credentials: [credentials], actions: [`Read:${Bucket}`, `Write:${Bucket}`, `List:${Bucket}`] }
|
||||
] }));
|
||||
// Seaweed's credential JSON uses accessKey/secretKey rather than SDK names.
|
||||
const auth = JSON.parse(await readFile(join(root, 'config/s3.json'), 'utf8'));
|
||||
for (const identity of auth.identities) identity.credentials = identity.credentials.map(c => ({ accessKey: c.accessKeyId, secretKey: c.secretAccessKey }));
|
||||
await writeFile(join(root, 'config/s3.json'), JSON.stringify(auth));
|
||||
run(`${prefix}-source`, '-p', '127.0.0.1::9000', '-e', `MINIO_ROOT_USER=${credentials.accessKeyId}`,
|
||||
'-e', `MINIO_ROOT_PASSWORD=${credentials.secretAccessKey}`, process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07',
|
||||
'server', '/data', '--console-address', ':9001');
|
||||
const sourceConfig = config(endpoint(`${prefix}-source`));
|
||||
source = client(sourceConfig);
|
||||
await ready(source);
|
||||
await call(source, 'CreateBucket', { Bucket });
|
||||
await contract(source, sourceConfig, Bucket);
|
||||
const targetConfig = config(await weed(`${prefix}-target`, join(root, 'target')));
|
||||
target = client(targetConfig);
|
||||
admin = client({ ...targetConfig, credentials: adminCredentials });
|
||||
await ready(admin);
|
||||
await call(admin, 'CreateBucket', { Bucket });
|
||||
await call(admin, 'CreateBucket', { Bucket: 'foreign-private' });
|
||||
await contract(target, targetConfig, Bucket, { foreignBucket: 'foreign-private' });
|
||||
// Confirm unauthenticated management ports are not reachable through the container IP.
|
||||
const ip = JSON.parse(docker('inspect', `${prefix}-target`))[0].NetworkSettings.Networks.bridge.IPAddress;
|
||||
for (const port of [9333, 19333, 8888, 18888, 8080, 18080, 8181, 9101]) {
|
||||
assert.throws(() => docker('exec', `${prefix}-source`, 'node', '-e',
|
||||
`const s=require('node:net').connect(${port},${JSON.stringify(ip)});s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));s.setTimeout(1500,()=>process.exit(1))`));
|
||||
}
|
||||
for (let i = 0; i < 1003; i++) await call(source, 'PutObject', { Bucket, Key: `tenant/filesbyid/${i}/ä + % #.txt`,
|
||||
Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } });
|
||||
await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' });
|
||||
await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES,
|
||||
Body: Readable.from((function* () { for (let i = 0; i < 256; i++) yield Buffer.alloc(1024 * 1024, i); })()),
|
||||
ContentType: 'application/zip', ContentDisposition: 'attachment; filename="large.zip"',
|
||||
CacheControl: 'private', ContentLanguage: 'de', Expires: new Date('2030-01-01T00:00:00Z') });
|
||||
const before = await inventory(source, Bucket);
|
||||
assert.equal(before.count, 1005);
|
||||
await call(source, 'PutObject', { Bucket, Key: 'unsupported-marker/', Body: '' });
|
||||
await assert.rejects(inventory(source, Bucket), /Schlüssel/);
|
||||
await call(source, 'DeleteObject', { Bucket, Key: 'unsupported-marker/' });
|
||||
const start = Date.now();
|
||||
const report = await transfer(source, target, Bucket, join(root, 'work'));
|
||||
await call(target, 'PutObject', { Bucket, Key: 'empty', Body: 'corrupt' });
|
||||
await assert.rejects(transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true }), /Integritätsfehler/);
|
||||
await transfer(source, target, Bucket, join(root, 'work')); // Resume repairs and re-reads, never trusts journal alone.
|
||||
await transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true });
|
||||
docker('stop', '-t', '60', `${prefix}-target`);
|
||||
docker('run', '--rm', '--pull', 'never', '-v', `${root}:/test`, '--entrypoint', '/bin/sh', spec.image,
|
||||
'-ec', 'tar -cpf /test/backup.tar -C /test/target .; mkdir /test/restored; tar -xpf /test/backup.tar -C /test/restored');
|
||||
const restoredConfig = config(await weed(`${prefix}-restored`, join(root, 'restored')));
|
||||
const restored = client(restoredConfig);
|
||||
const restoredAdmin = client({ ...restoredConfig, credentials: adminCredentials });
|
||||
try {
|
||||
await ready(restoredAdmin);
|
||||
await verifyManifest(restored, Bucket, report);
|
||||
await contract(restored, restoredConfig, Bucket, { foreignBucket: 'foreign-private' });
|
||||
} finally { restored.destroy(); restoredAdmin.destroy(); }
|
||||
console.log(JSON.stringify({ result: 'passed', objects: report.count, bytes: report.bytes,
|
||||
seconds: (Date.now() - start) / 1000, seaweed: spec.image, source: docker('image', 'inspect',
|
||||
process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', '--format', '{{.Id}}'),
|
||||
evidence: root }, null, 2));
|
||||
} catch (error) {
|
||||
for (const name of containers) { try { console.error(docker('logs', '--tail', '12', name)); } catch {} }
|
||||
throw error;
|
||||
} finally {
|
||||
source?.destroy(); target?.destroy(); admin?.destroy();
|
||||
for (const name of containers.reverse()) { try { docker('rm', '-f', name); } catch {} }
|
||||
// Preserve the isolated backup and report for inspection; no production paths are touched.
|
||||
}
|
||||
Reference in New Issue
Block a user