flfeders/FEDEO
2
1
Fork 0
Code Issues 116 Pull Requests Actions Packages Projects 23 Releases 20 Wiki Activity

fix(storage): verify migration of exports up to one GiB

Browse Source
This commit is contained in:
Florian Federspiel
2026-09-28 14:08:59 +02:00
parent e9b2e3e229
commit a3f4a7c004
4 changed files with 32 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
scripts/storage/tests/integration.mjs
View File

@@ -1,6 +1,6 @@
// Explicit integration run; never discovers or connects to an existing FEDEO stack.
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { execFileSync, spawnSync } from 'node:child_process';
import { mkdtemp, mkdir, writeFile, readFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -28,6 +28,20 @@ async function ready(s3) {
}
throw new Error('S3 did not become ready');
}
async function restoredContentsReady(s3, bucket, report) {
// Like the host helper, wait for volume registration after a cold start.
// ListBuckets can already succeed while reading object data still returns 500.
const deadline = Date.now() + 180_000;
for (;;) {
try { return await verifyManifest(s3, bucket, report); }
catch (error) {
const transient = [500, 502, 503, 504].includes(error.$metadata?.httpStatusCode) ||
['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EHOSTUNREACH'].includes(error.code);
if (!transient || Date.now() >= deadline) throw error;
await new Promise(resolve => setTimeout(resolve, 2000));
}
}
}
async function weed(name, data) {
await mkdir(join(data, 'master'), { recursive: true });
run(name, '-p', '127.0.0.1::9000', '-v', `${data}:/data`, '-v', `${root}/config:/etc/fedeo:ro`,
@@ -73,11 +87,12 @@ try {
Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } });
await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' });
await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES,
Body: Readable.from((function* () { for (let i = 0; i < 256; i++) yield Buffer.alloc(1024 * 1024, i); })()),
Body: Readable.from((function* () { for (let i = 0; i < MAX_OBJECT_BYTES / (1024 * 1024); i++) yield Buffer.alloc(1024 * 1024, i % 256); })()),
ContentType: 'application/zip', ContentDisposition: 'attachment; filename="large.zip"',
CacheControl: 'private', ContentLanguage: 'de', Expires: new Date('2030-01-01T00:00:00Z') });
const before = await inventory(source, Bucket);
assert.equal(before.count, 1005);
assert.equal(before.objects.find(o => o.Key === 'large.zip').Size, MAX_OBJECT_BYTES);
await call(source, 'PutObject', { Bucket, Key: 'unsupported-marker/', Body: '' });
await assert.rejects(inventory(source, Bucket), /Schlüssel/);
await call(source, 'DeleteObject', { Bucket, Key: 'unsupported-marker/' });
@@ -95,7 +110,7 @@ try {
const restoredAdmin = client({ ...restoredConfig, credentials: adminCredentials });
try {
await ready(restoredAdmin);
await verifyManifest(restored, Bucket, report);
await restoredContentsReady(restored, Bucket, report);
await contract(restored, restoredConfig, Bucket, { foreignBucket: 'foreign-private' });
} finally { restored.destroy(); restoredAdmin.destroy(); }
console.log(JSON.stringify({ result: 'passed', objects: report.count, bytes: report.bytes,
@@ -103,7 +118,10 @@ try {
process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', '--format', '{{.Id}}'),
evidence: root }, null, 2));
} catch (error) {
for (const name of containers) { try { console.error(docker('logs', '--tail', '12', name)); } catch {} }
for (const name of containers) {
const logs = spawnSync('docker', ['logs', '--tail', '30', name], { encoding: 'utf8' });
console.error(logs.stdout + logs.stderr);
}
throw error;
} finally {
source?.destroy(); target?.destroy(); admin?.destroy();