feat(storage): add verified offline MinIO migration pilot

2026-09-27 17:22:13 +02:00
parent 0005bfc4cf
commit 46ee819029
22 changed files with 3527 additions and 0 deletions
--- a/scripts/storage/tests/Dockerfile.minio
+++ b/scripts/storage/tests/Dockerfile.minio
@@ -0,0 +1,6 @@
+# Test fixture only. The pilot bundle uses the instance's already installed MinIO.
+FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32
+RUN wget -q -O /usr/local/bin/minio https://github.com/minio/minio/releases/download/RELEASE.2025-09-07T16-13-09Z/minio.linux-amd64.RELEASE.2025-09-07T16-13-09Z \
+    && echo '7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f  /usr/local/bin/minio' | sha256sum -c - \
+    && chmod 755 /usr/local/bin/minio
+ENTRYPOINT ["minio"]
--- a/scripts/storage/tests/host_integration.py
+++ b/scripts/storage/tests/host_integration.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Real Docker host-state test with synthetic services, never a real FEDEO instance."""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import uuid
+import sys
+
+ROOT = Path(__file__).resolve().parents[2]
+spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+def docker(*args):
+    return subprocess.check_output(["docker", *map(str, args)], text=True, stderr=subprocess.PIPE).strip()
+
+
+def fixture():
+    root = Path(tempfile.mkdtemp(prefix="fedeo-host-migration-"))
+    project = "fedeo-pilot-" + uuid.uuid4().hex[:8]
+    (root / "minio").mkdir()
+    (root / "postgres").mkdir()
+    (root / ".env").write_text("# synthetic test only\n")
+    secret = "test-with-$-interpolation"
+    env = {"S3_ENDPOINT": "http://minio:9000", "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo",
+           "S3_ACCESS_KEY": "test-user", "S3_SECRET_KEY": secret, "FEDEO_VERSION": "3.3.0",
+           "DATABASE_URL": "postgres://fedeo:test-db@db:5432/fedeo", "FEDEO_FILE_BACKEND": "s3"}
+    idle = {"image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
+            "restart": "unless-stopped", "networks": ["internal"],
+            "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(0)"], "interval": "1s"}}
+    import copy
+    services = {n: copy.deepcopy(idle) for n in ("backend", "frontend", "traefik", "node-exporter", "createbuckets")}
+    services["createbuckets"].update(command=["node", "-e", "process.exit(0)"], restart="no")
+    services["backend"]["environment"] = env
+    services["backend"]["image"] = "fedeo-storage-tools:pilot"
+    services["backend"]["entrypoint"] = ["node"]
+    services["backend"]["command"] = ["-e", "setInterval(()=>{},1000)"]
+    services["backend"]["depends_on"] = {"minio": {"condition": "service_healthy"}, "db": {"condition": "service_healthy"},
+                                                 "createbuckets": {"condition": "service_completed_successfully"}}
+    services["minio"] = {"image": "fedeo-minio-fixture:2025-09-07", "restart": "unless-stopped",
+        "command": ["server", "/data", "--console-address", ":9001"], "networks": ["internal"],
+        "volumes": [str(root / "minio") + ":/data"],
+        "environment": {"MINIO_ROOT_USER": "test-user", "MINIO_ROOT_PASSWORD": secret},
+        "healthcheck": {"test": ["CMD", "node", "-e", "fetch('http://127.0.0.1:9000/minio/health/live').then(r=>process.exit(r.ok?0:1))"], "interval": "1s"}}
+    services["db"] = {"image": "postgres:16-alpine", "restart": "unless-stopped", "networks": ["internal"],
+        "volumes": [str(root / "postgres") + ":/var/lib/postgresql/data"],
+        "environment": {"POSTGRES_USER": "fedeo", "POSTGRES_PASSWORD": "test-db", "POSTGRES_DB": "fedeo"},
+        "healthcheck": {"test": ["CMD", "pg_isready", "-U", "fedeo"], "interval": "1s"}}
+    config = {"name": project, "services": services, "networks": {"internal": {"name": project + "-internal"}}}
+    (root / "docker-compose.yml").write_text(json.dumps(m.literal(config)))
+    return root, project
+
+
+def seed(root, project, hidden=False):
+    code = """import { client, call } from './s3.mjs';
+const s=client({endpoint:'http://minio:9000',region:'eu-central-1',credentials:{accessKeyId:'test-user',secretAccessKey:'test-with-$-interpolation'}});
+await call(s,'CreateBucket',{Bucket:'fedeo'});
+await call(s,'PutObject',{Bucket:'fedeo',Key:'tenant/filesbyid/1/ä + % #.pdf',Body:Buffer.from('fixture'),ContentType:'application/pdf'});"""
+    if hidden:
+        code += "for(const Key of ['hidden','hidden/child']) await call(s,'PutObject',{Bucket:'fedeo',Key,Body:'fixture'});"
+    code += "s.destroy();"
+    docker("run", "--rm", "--pull", "never", "--network", project + "-internal", "--entrypoint", "node", "fedeo-storage-tools:pilot", "--input-type=module", "-e", code)
+
+
+def run(case):
+    root, project = fixture()
+    base = ["compose", "-p", project, "-f", str(root / "docker-compose.yml")]
+    try:
+        docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
+        seed(root, project, hidden=case == "hidden-prefix")
+        migration = m.Migration(root, "fedeo-storage-tools:pilot")
+        with migration.engine.lock():
+            if case == "hidden-prefix":
+                try:
+                    migration.plan()
+                    raise AssertionError("hidden key conflict must block planning")
+                except m.Error as error:
+                    assert "konflikt" in str(error)
+                assert not (root / ".updates/status.json").exists()
+                print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
+                return
+            migration.plan()
+            assert migration.state["count"] == 1
+            migration.migrate(True)
+            assert migration.state["phase"] == "verified"
+            assert m.u.read_json(root / ".updates/status.json")["recoveryRequired"]
+            # Simulate restart by reconstructing the process and persisted state.
+            migration = m.Migration(root)
+            if case == "rollback":
+                migration.rollback()
+                assert migration.state["phase"] == "rolled_back"
+                assert not m.u.read_json(root / ".updates/status.json").get("recoveryRequired")
+                assert migration.inspect(migration.state["containers"]["minio"]["id"])["State"]["Running"]
+            else:
+                migration.activate()
+                assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
+                assert migration.state["restoreVerifiedAt"]
+                old = migration.inspect(migration.state["containers"]["minio"]["id"])
+                assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
+                assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
+                try:
+                    migration.rollback()
+                    raise AssertionError("rollback after writes must fail")
+                except m.Error:
+                    pass
+        print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
+    except BaseException:
+        print("Failure evidence: " + str(root), flush=True)
+        log = root / ".updates/storage-migration/commands.log"
+        if log.exists():
+            print(log.read_text()[-8000:], flush=True)
+        ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project,
+                     "--filter", "label=com.docker.compose.service=objectstore").split()
+        for cid in ids:
+            logs = subprocess.run(["docker", "logs", "--tail", "60", cid], capture_output=True, text=True)
+            print(logs.stdout + logs.stderr, flush=True)
+        raise
+    finally:
+        ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split()
+        if ids:
+            docker("rm", "-f", *ids)
+        remaining = docker("ps", "-aq", "--filter", "label=io.fedeo.storage-job=" + str(root / ".updates/storage-migration")).split()
+        if remaining:
+            docker("rm", "-f", *remaining)
+        docker("network", "rm", project + "-internal")
+
+
+if __name__ == "__main__":
+    for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
+        assert case in ("hidden-prefix", "rollback", "activate")
+        run(case)
--- a/scripts/storage/tests/integration.mjs
+++ b/scripts/storage/tests/integration.mjs
@@ -0,0 +1,112 @@
+// Explicit integration run; never discovers or connects to an existing FEDEO stack.
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtemp, mkdir, writeFile, readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { randomUUID, createHash } from 'node:crypto';
+import { client, call, contract, inventory, transfer, verifyManifest, MAX_OBJECT_BYTES } from '../s3.mjs';
+
+const docker = (...args) => execFileSync('docker', args, { encoding: 'utf8', timeout: 120_000, stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+const spec = JSON.parse(await readFile(new URL('../seaweedfs.json', import.meta.url), 'utf8'));
+const root = await mkdtemp(join(tmpdir(), 'fedeo-storage-contract-'));
+const prefix = `fedeo-contract-${randomUUID().slice(0, 8)}`;
+const containers = [];
+const Bucket = 'fedeo';
+const credentials = { accessKeyId: 'fedeo-test-user', secretAccessKey: 'fedeo-test-secret-only' };
+const adminCredentials = { accessKeyId: 'fedeo-test-admin', secretAccessKey: 'fedeo-test-admin-secret' };
+process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED';
+const config = endpoint => ({ endpoint, region: 'eu-central-1', credentials });
+let source, target, admin;
+function run(name, ...args) { containers.push(name); return docker('run', '-d', '--pull', 'never', '--name', name, ...args); }
+function endpoint(name) { return 'http://' + docker('port', name, '9000/tcp'); }
+async function ready(s3) {
+  for (let i = 0; i < 90; i++) {
+    try { await call(s3, 'ListBuckets', {}); return; }
+    catch { await new Promise(resolve => setTimeout(resolve, 1000)); }
+  }
+  throw new Error('S3 did not become ready');
+}
+async function weed(name, data) {
+  await mkdir(join(data, 'master'), { recursive: true });
+  run(name, '-p', '127.0.0.1::9000', '-v', `${data}:/data`, '-v', `${root}/config:/etc/fedeo:ro`,
+    '-e', 'FEDEO_S3_CONFIG_SHA256=' + createHash('sha256').update(await readFile(join(root, 'config/s3.json'))).digest('hex'),
+    '--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command);
+  return endpoint(name);
+}
+try {
+  await mkdir(join(root, 'config'));
+  await writeFile(join(root, 'config/start-seaweedfs.sh'), await readFile(new URL('../start-seaweedfs.sh', import.meta.url)));
+  assert.throws(() => docker('run', '--rm', '--pull', 'never', '-v', `${root}/config:/etc/fedeo:ro`,
+    '--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command));
+  await writeFile(join(root, 'config/s3.json'), JSON.stringify({ identities: [
+    { name: 'admin', credentials: [adminCredentials], actions: ['Admin', 'Read', 'Write', 'List', 'Tagging'] },
+    { name: 'fedeo', credentials: [credentials], actions: [`Read:${Bucket}`, `Write:${Bucket}`, `List:${Bucket}`] }
+  ] }));
+  // Seaweed's credential JSON uses accessKey/secretKey rather than SDK names.
+  const auth = JSON.parse(await readFile(join(root, 'config/s3.json'), 'utf8'));
+  for (const identity of auth.identities) identity.credentials = identity.credentials.map(c => ({ accessKey: c.accessKeyId, secretKey: c.secretAccessKey }));
+  await writeFile(join(root, 'config/s3.json'), JSON.stringify(auth));
+  run(`${prefix}-source`, '-p', '127.0.0.1::9000', '-e', `MINIO_ROOT_USER=${credentials.accessKeyId}`,
+    '-e', `MINIO_ROOT_PASSWORD=${credentials.secretAccessKey}`, process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07',
+    'server', '/data', '--console-address', ':9001');
+  const sourceConfig = config(endpoint(`${prefix}-source`));
+  source = client(sourceConfig);
+  await ready(source);
+  await call(source, 'CreateBucket', { Bucket });
+  await contract(source, sourceConfig, Bucket);
+  const targetConfig = config(await weed(`${prefix}-target`, join(root, 'target')));
+  target = client(targetConfig);
+  admin = client({ ...targetConfig, credentials: adminCredentials });
+  await ready(admin);
+  await call(admin, 'CreateBucket', { Bucket });
+  await call(admin, 'CreateBucket', { Bucket: 'foreign-private' });
+  await contract(target, targetConfig, Bucket, { foreignBucket: 'foreign-private' });
+  // Confirm unauthenticated management ports are not reachable through the container IP.
+  const ip = JSON.parse(docker('inspect', `${prefix}-target`))[0].NetworkSettings.Networks.bridge.IPAddress;
+  for (const port of [9333, 19333, 8888, 18888, 8080, 18080, 8181, 9101]) {
+    assert.throws(() => docker('exec', `${prefix}-source`, 'node', '-e',
+      `const s=require('node:net').connect(${port},${JSON.stringify(ip)});s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));s.setTimeout(1500,()=>process.exit(1))`));
+  }
+  for (let i = 0; i < 1003; i++) await call(source, 'PutObject', { Bucket, Key: `tenant/filesbyid/${i}/ä + % #.txt`,
+    Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } });
+  await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' });
+  await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES,
+    Body: Readable.from((function* () { for (let i = 0; i < 256; i++) yield Buffer.alloc(1024 * 1024, i); })()),
+    ContentType: 'application/zip', ContentDisposition: 'attachment; filename="large.zip"',
+    CacheControl: 'private', ContentLanguage: 'de', Expires: new Date('2030-01-01T00:00:00Z') });
+  const before = await inventory(source, Bucket);
+  assert.equal(before.count, 1005);
+  await call(source, 'PutObject', { Bucket, Key: 'unsupported-marker/', Body: '' });
+  await assert.rejects(inventory(source, Bucket), /Schlüssel/);
+  await call(source, 'DeleteObject', { Bucket, Key: 'unsupported-marker/' });
+  const start = Date.now();
+  const report = await transfer(source, target, Bucket, join(root, 'work'));
+  await call(target, 'PutObject', { Bucket, Key: 'empty', Body: 'corrupt' });
+  await assert.rejects(transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true }), /Integritätsfehler/);
+  await transfer(source, target, Bucket, join(root, 'work')); // Resume repairs and re-reads, never trusts journal alone.
+  await transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true });
+  docker('stop', '-t', '60', `${prefix}-target`);
+  docker('run', '--rm', '--pull', 'never', '-v', `${root}:/test`, '--entrypoint', '/bin/sh', spec.image,
+    '-ec', 'tar -cpf /test/backup.tar -C /test/target .; mkdir /test/restored; tar -xpf /test/backup.tar -C /test/restored');
+  const restoredConfig = config(await weed(`${prefix}-restored`, join(root, 'restored')));
+  const restored = client(restoredConfig);
+  const restoredAdmin = client({ ...restoredConfig, credentials: adminCredentials });
+  try {
+    await ready(restoredAdmin);
+    await verifyManifest(restored, Bucket, report);
+    await contract(restored, restoredConfig, Bucket, { foreignBucket: 'foreign-private' });
+  } finally { restored.destroy(); restoredAdmin.destroy(); }
+  console.log(JSON.stringify({ result: 'passed', objects: report.count, bytes: report.bytes,
+    seconds: (Date.now() - start) / 1000, seaweed: spec.image, source: docker('image', 'inspect',
+      process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', '--format', '{{.Id}}'),
+    evidence: root }, null, 2));
+} catch (error) {
+  for (const name of containers) { try { console.error(docker('logs', '--tail', '12', name)); } catch {} }
+  throw error;
+} finally {
+  source?.destroy(); target?.destroy(); admin?.destroy();
+  for (const name of containers.reverse()) { try { docker('rm', '-f', name); } catch {} }
+  // Preserve the isolated backup and report for inspection; no production paths are touched.
+}
--- a/scripts/storage/tests/keys.test.mjs
+++ b/scripts/storage/tests/keys.test.mjs
@@ -0,0 +1,37 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { validateKeys, MAX_OBJECT_BYTES, metadata } from '../s3.mjs';
+import { cleanupProbe, atomicJson } from '../s3.mjs';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+test('accepts FEDEO keys, special characters, empty objects and size boundary', () => {
+  validateKeys([{ Key: 'tenant/filesbyid/123/ä + %#.pdf', Size: 0 }, { Key: 'export.zip', Size: MAX_OBJECT_BYTES }]);
+});
+test('blocks lossy paths and file/directory collisions regardless of ordering', () => {
+  for (const Key of ['/a', 'a/', 'a//b', 'a/../b', './a', 'a\\b', 'a\nb'])
+    assert.throws(() => validateKeys([{ Key, Size: 1 }]));
+  assert.throws(() => validateKeys([{ Key: 'a/b', Size: 1 }, { Key: 'a', Size: 1 }]));
+  assert.throws(() => validateKeys([{ Key: 'a', Size: MAX_OBJECT_BYTES + 1 }]));
+});
+test('rejects encryption and nonportable attributes rather than dropping them', () => {
+  for (const value of [{ ServerSideEncryption: 'AES256' }, { ObjectLockMode: 'GOVERNANCE' },
+    { StorageClass: 'GLACIER' }, { WebsiteRedirectLocation: '/other' }]) assert.throws(() => metadata(value));
+  assert.deepEqual(metadata({ ContentType: 'application/pdf', Metadata: { custom: 'value' } }),
+    { ContentType: 'application/pdf', Metadata: { custom: 'value' } });
+});
+
+test('interrupted contract cleanup deletes only its two journalled probe objects', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'fedeo-probe-test-'));
+  try {
+    const path = join(directory, 'probe.json');
+    const key = 'fedeo-contract-11111111-1111-1111-1111-111111111111/test';
+    await atomicJson(path, { key });
+    const inputs = [];
+    await cleanupProbe({ send: async command => { inputs.push(command.input); return {}; } }, 'bucket', path);
+    assert.deepEqual(inputs, [{ Bucket: 'bucket', Key: key }, { Bucket: 'bucket', Key: key + '.stream' }]);
+    await atomicJson(path, { key: 'real-business-file.pdf' });
+    await assert.rejects(cleanupProbe({ send: async () => { throw new Error('must not send'); } }, 'bucket', path), /Probejournal/);
+  } finally { await rm(directory, { recursive: true, force: true }); }
+});
--- a/scripts/storage/tests/test_host.py
+++ b/scripts/storage/tests/test_host.py
@@ -0,0 +1,125 @@
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import Mock
+
+spec = importlib.util.spec_from_file_location("storage_migration", Path(__file__).resolve().parents[2] / "selfhost-storage-migrate.py")
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+class HostSafetyTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        for name in ("minio", "postgres"):
+            (self.root / name).mkdir()
+        self.config = {"services": {n: {} for n in ("db", "minio", "createbuckets", "backend", "frontend", "traefik", "node-exporter")}}
+        for name, path, target in (("minio", "minio", "/data"), ("db", "postgres", "/var/lib/postgresql/data")):
+            self.config["services"][name]["volumes"] = [{"type": "bind", "source": str(self.root / path), "target": target}]
+        self.config["services"]["minio"].update(networks={"internal": {}}, environment={"MINIO_ROOT_USER": "user", "MINIO_ROOT_PASSWORD": "secret"})
+        self.config["services"]["db"]["environment"] = {"POSTGRES_USER": "u", "POSTGRES_PASSWORD": "p", "POSTGRES_DB": "db"}
+        self.config["services"]["backend"]["environment"] = {"DATABASE_URL": "postgres://u:p@db:5432/db", "S3_ENDPOINT": "http://minio:9000",
+            "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo", "S3_ACCESS_KEY": "user", "S3_SECRET_KEY": "secret"}
+
+    def test_rejects_external_or_unbacked_storage_and_secret_drift(self):
+        m.validate_config(self.config, self.root)
+        mutations = [lambda c: c["services"]["minio"].update(ports=["9000:9000"]),
+            lambda c: c["services"]["backend"]["environment"].update(S3_ENDPOINT="https://external.example"),
+            lambda c: c["services"]["backend"]["environment"].update(S3_SECRET_KEY="different"),
+            lambda c: c["services"]["minio"]["volumes"][0].update(source="/elsewhere"),
+            lambda c: c["services"].update(unknown={}),
+            lambda c: c["services"]["backend"]["environment"].update(DATABASE_URL="postgres://u:p@db:5432/other")]
+        for mutate in mutations:
+            config = copy.deepcopy(self.config)
+            mutate(config)
+            with self.assertRaises(m.Error):
+                m.validate_config(config, self.root)
+
+    def test_rejects_symlink_source(self):
+        (self.root / "minio").rmdir()
+        (self.root / "minio").symlink_to(self.root / "postgres")
+        with self.assertRaises(m.Error):
+            m.validate_config(self.config, self.root)
+
+    def test_cold_index_detects_objects_hidden_by_minio_listing(self):
+        source = self.root / "minio"
+        (source / ".minio.sys").mkdir()
+        (source / ".minio.sys/format.json").write_text(json.dumps({"format": "xl-single", "xl": {"version": "3"}}))
+        (source / "fedeo/a").mkdir(parents=True)
+        (source / "fedeo/a/xl.meta").write_bytes(b"opaque")
+        m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}]}, exact=True)
+        (source / "fedeo/a/b").mkdir()
+        (source / "fedeo/a/b/xl.meta").write_bytes(b"hidden-from-S3-LIST")
+        with self.assertRaisesRegex(m.Error, "Konflikt|konflikt"):
+            m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}]}, exact=True)
+        (source / "fedeo/a/b/xl.meta").unlink()
+        with self.assertRaisesRegex(m.Error, "weichen ab"):
+            m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}, {"Key": "missing"}]}, exact=True)
+
+    def test_update_and_migration_share_exclusive_lock(self):
+        a, b = m.Migration(self.root), m.Migration(self.root)
+        with a.engine.lock():
+            with self.assertRaises(m.Error):
+                with b.engine.lock():
+                    self.fail("lock must be exclusive")
+
+    def test_failed_freeze_keeps_persistent_recovery_guard(self):
+        migration = m.Migration(self.root)
+        migration.state = {"startedAt": "test", "writesReleased": False, "containers": {"backend": {"id": "backend"}}}
+        def fail(*args, **kwargs):
+            state = m.u.read_json(migration.engine.home / "status.json")
+            self.assertTrue(state["recoveryRequired"])
+            self.assertEqual(m.u.read_json(migration.home / "state.json")["phase"], "freezing")
+            raise m.Error("power loss")
+        migration.docker = fail
+        with self.assertRaises(m.Error):
+            migration.freeze()
+        self.assertFalse(m.u.read_json(migration.home / "state.json")["writesReleased"])
+
+    def test_backup_corruption_prevents_progress(self):
+        migration = m.Migration(self.root)
+        backup = migration.home / "backup"
+        backup.mkdir()
+        (backup / "files.tar").write_bytes(b"valid")
+        m.u.write_json(backup / "complete.json", {"sha256": {"files.tar": m.sha_file(backup / "files.tar")}})
+        migration.check_backup()
+        (backup / "files.tar").write_bytes(b"corrupt")
+        with self.assertRaises(m.Error):
+            migration.check_backup()
+
+    def test_start_failure_is_already_past_rollback_boundary(self):
+        migration = m.Migration(self.root)
+        migration.state = {"startedAt": "test", "phase": "verified", "network": "internal", "writesReleased": False,
+                           "containers": {"minio": {"id": "old"}}}
+        for method in ("assert_frozen", "verify_original", "check_backup", "cleanup_helper", "helper", "docker", "compose"):
+            setattr(migration, method, Mock())
+        migration.inspect = lambda _: {"NetworkSettings": {"Networks": {}}}
+        migration.restore_test = Mock()
+        migration.runner = Mock()
+        (migration.home / "target.json").write_text('{}')
+        def fail_start():
+            self.assertTrue(m.u.read_json(migration.home / "state.json")["writesReleased"])
+            raise m.Error("backend did not start")
+        migration.start_application = fail_start
+        from unittest.mock import patch
+        (migration.home / "credentials.json").write_text('{"bucket":"fedeo"}')
+        with patch.object(m, "validate_minio_listing"):
+            (migration.home / "copy").mkdir()
+            (migration.home / "copy/inventory.json").write_text('{}')
+            with self.assertRaises(m.Error):
+                migration.activate()
+        with self.assertRaises(m.Error):
+            migration.rollback()
+
+    def test_compose_roundtrip_preserves_dollar_signs(self):
+        value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]}
+        self.assertEqual(m.resolved(m.literal(value)), value)
+
+
+if __name__ == "__main__":
+    unittest.main()