131 lines
8.4 KiB
JavaScript
131 lines
8.4 KiB
JavaScript
// Explicit integration run; never discovers or connects to an existing FEDEO stack.
|
|
import assert from 'node:assert/strict';
|
|
import { execFileSync, spawnSync } from 'node:child_process';
|
|
import { mkdtemp, mkdir, writeFile, readFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { Readable } from 'node:stream';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { client, call, contract, inventory, transfer, verifyManifest, MAX_OBJECT_BYTES } from '../s3.mjs';
|
|
|
|
const docker = (...args) => execFileSync('docker', args, { encoding: 'utf8', timeout: 120_000, stdio: ['ignore', 'pipe', 'pipe'] }).trim();
|
|
const spec = JSON.parse(await readFile(new URL('../seaweedfs.json', import.meta.url), 'utf8'));
|
|
const root = await mkdtemp(join(tmpdir(), 'fedeo-storage-contract-'));
|
|
const prefix = `fedeo-contract-${randomUUID().slice(0, 8)}`;
|
|
const containers = [];
|
|
const Bucket = 'fedeo';
|
|
const credentials = { accessKeyId: 'fedeo-test-user', secretAccessKey: 'fedeo-test-secret-only' };
|
|
const adminCredentials = { accessKeyId: 'fedeo-test-admin', secretAccessKey: 'fedeo-test-admin-secret' };
|
|
process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED';
|
|
const config = endpoint => ({ endpoint, region: 'eu-central-1', credentials });
|
|
let source, target, admin;
|
|
function run(name, ...args) { containers.push(name); return docker('run', '-d', '--pull', 'never', '--name', name, ...args); }
|
|
function endpoint(name) { return 'http://' + docker('port', name, '9000/tcp'); }
|
|
async function ready(s3) {
|
|
for (let i = 0; i < 90; i++) {
|
|
try { await call(s3, 'ListBuckets', {}); return; }
|
|
catch { await new Promise(resolve => setTimeout(resolve, 1000)); }
|
|
}
|
|
throw new Error('S3 did not become ready');
|
|
}
|
|
async function restoredContentsReady(s3, bucket, report) {
|
|
// Like the host helper, wait for volume registration after a cold start.
|
|
// ListBuckets can already succeed while reading object data still returns 500.
|
|
const deadline = Date.now() + 180_000;
|
|
for (;;) {
|
|
try { return await verifyManifest(s3, bucket, report); }
|
|
catch (error) {
|
|
const transient = [500, 502, 503, 504].includes(error.$metadata?.httpStatusCode) ||
|
|
['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EHOSTUNREACH'].includes(error.code);
|
|
if (!transient || Date.now() >= deadline) throw error;
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
}
|
|
}
|
|
}
|
|
async function weed(name, data) {
|
|
await mkdir(join(data, 'master'), { recursive: true });
|
|
run(name, '-p', '127.0.0.1::9000', '-v', `${data}:/data`, '-v', `${root}/config:/etc/fedeo:ro`,
|
|
'-e', 'FEDEO_S3_CONFIG_SHA256=' + createHash('sha256').update(await readFile(join(root, 'config/s3.json'))).digest('hex'),
|
|
'--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command);
|
|
return endpoint(name);
|
|
}
|
|
try {
|
|
await mkdir(join(root, 'config'));
|
|
await writeFile(join(root, 'config/start-seaweedfs.sh'), await readFile(new URL('../start-seaweedfs.sh', import.meta.url)));
|
|
assert.throws(() => docker('run', '--rm', '--pull', 'never', '-v', `${root}/config:/etc/fedeo:ro`,
|
|
'--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command));
|
|
await writeFile(join(root, 'config/s3.json'), JSON.stringify({ identities: [
|
|
{ name: 'admin', credentials: [adminCredentials], actions: ['Admin', 'Read', 'Write', 'List', 'Tagging'] },
|
|
{ name: 'fedeo', credentials: [credentials], actions: [`Read:${Bucket}`, `Write:${Bucket}`, `List:${Bucket}`] }
|
|
] }));
|
|
// Seaweed's credential JSON uses accessKey/secretKey rather than SDK names.
|
|
const auth = JSON.parse(await readFile(join(root, 'config/s3.json'), 'utf8'));
|
|
for (const identity of auth.identities) identity.credentials = identity.credentials.map(c => ({ accessKey: c.accessKeyId, secretKey: c.secretAccessKey }));
|
|
await writeFile(join(root, 'config/s3.json'), JSON.stringify(auth));
|
|
run(`${prefix}-source`, '-p', '127.0.0.1::9000', '-e', `MINIO_ROOT_USER=${credentials.accessKeyId}`,
|
|
'-e', `MINIO_ROOT_PASSWORD=${credentials.secretAccessKey}`, process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07',
|
|
'server', '/data', '--console-address', ':9001');
|
|
const sourceConfig = config(endpoint(`${prefix}-source`));
|
|
source = client(sourceConfig);
|
|
await ready(source);
|
|
await call(source, 'CreateBucket', { Bucket });
|
|
await contract(source, sourceConfig, Bucket);
|
|
const targetConfig = config(await weed(`${prefix}-target`, join(root, 'target')));
|
|
target = client(targetConfig);
|
|
admin = client({ ...targetConfig, credentials: adminCredentials });
|
|
await ready(admin);
|
|
await call(admin, 'CreateBucket', { Bucket });
|
|
await call(admin, 'CreateBucket', { Bucket: 'foreign-private' });
|
|
await contract(target, targetConfig, Bucket, { foreignBucket: 'foreign-private' });
|
|
// Confirm unauthenticated management ports are not reachable through the container IP.
|
|
const ip = JSON.parse(docker('inspect', `${prefix}-target`))[0].NetworkSettings.Networks.bridge.IPAddress;
|
|
for (const port of [9333, 19333, 8888, 18888, 8080, 18080, 8181, 9101]) {
|
|
assert.throws(() => docker('exec', `${prefix}-source`, 'node', '-e',
|
|
`const s=require('node:net').connect(${port},${JSON.stringify(ip)});s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));s.setTimeout(1500,()=>process.exit(1))`));
|
|
}
|
|
for (let i = 0; i < 1003; i++) await call(source, 'PutObject', { Bucket, Key: `tenant/filesbyid/${i}/ä + % #.txt`,
|
|
Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } });
|
|
await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' });
|
|
await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES,
|
|
Body: Readable.from((function* () { for (let i = 0; i < MAX_OBJECT_BYTES / (1024 * 1024); i++) yield Buffer.alloc(1024 * 1024, i % 256); })()),
|
|
ContentType: 'application/zip', ContentDisposition: 'attachment; filename="large.zip"',
|
|
CacheControl: 'private', ContentLanguage: 'de', Expires: new Date('2030-01-01T00:00:00Z') });
|
|
const before = await inventory(source, Bucket);
|
|
assert.equal(before.count, 1005);
|
|
assert.equal(before.objects.find(o => o.Key === 'large.zip').Size, MAX_OBJECT_BYTES);
|
|
await call(source, 'PutObject', { Bucket, Key: 'unsupported-marker/', Body: '' });
|
|
await assert.rejects(inventory(source, Bucket), /Schlüssel/);
|
|
await call(source, 'DeleteObject', { Bucket, Key: 'unsupported-marker/' });
|
|
const start = Date.now();
|
|
const report = await transfer(source, target, Bucket, join(root, 'work'));
|
|
await call(target, 'PutObject', { Bucket, Key: 'empty', Body: 'corrupt' });
|
|
await assert.rejects(transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true }), /Integritätsfehler/);
|
|
await transfer(source, target, Bucket, join(root, 'work')); // Resume repairs and re-reads, never trusts journal alone.
|
|
await transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true });
|
|
docker('stop', '-t', '60', `${prefix}-target`);
|
|
docker('run', '--rm', '--pull', 'never', '-v', `${root}:/test`, '--entrypoint', '/bin/sh', spec.image,
|
|
'-ec', 'tar -cpf /test/backup.tar -C /test/target .; mkdir /test/restored; tar -xpf /test/backup.tar -C /test/restored');
|
|
const restoredConfig = config(await weed(`${prefix}-restored`, join(root, 'restored')));
|
|
const restored = client(restoredConfig);
|
|
const restoredAdmin = client({ ...restoredConfig, credentials: adminCredentials });
|
|
try {
|
|
await ready(restoredAdmin);
|
|
await restoredContentsReady(restored, Bucket, report);
|
|
await contract(restored, restoredConfig, Bucket, { foreignBucket: 'foreign-private' });
|
|
} finally { restored.destroy(); restoredAdmin.destroy(); }
|
|
console.log(JSON.stringify({ result: 'passed', objects: report.count, bytes: report.bytes,
|
|
seconds: (Date.now() - start) / 1000, seaweed: spec.image, source: docker('image', 'inspect',
|
|
process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', '--format', '{{.Id}}'),
|
|
evidence: root }, null, 2));
|
|
} catch (error) {
|
|
for (const name of containers) {
|
|
const logs = spawnSync('docker', ['logs', '--tail', '30', name], { encoding: 'utf8' });
|
|
console.error(logs.stdout + logs.stderr);
|
|
}
|
|
throw error;
|
|
} finally {
|
|
source?.destroy(); target?.destroy(); admin?.destroy();
|
|
for (const name of containers.reverse()) { try { docker('rm', '-f', name); } catch {} }
|
|
// Preserve the isolated backup and report for inspection; no production paths are touched.
|
|
}
|