feat(storage): add verified offline MinIO migration pilot

scripts/storage/tests/host_integration.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Real Docker host-state test with synthetic services, never a real FEDEO instance."""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import uuid
+import sys
+
+ROOT = Path(__file__).resolve().parents[2]
+spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+def docker(*args):
+    return subprocess.check_output(["docker", *map(str, args)], text=True, stderr=subprocess.PIPE).strip()
+
+
+def fixture():
+    root = Path(tempfile.mkdtemp(prefix="fedeo-host-migration-"))
+    project = "fedeo-pilot-" + uuid.uuid4().hex[:8]
+    (root / "minio").mkdir()
+    (root / "postgres").mkdir()
+    (root / ".env").write_text("# synthetic test only\n")
+    secret = "test-with-$-interpolation"
+    env = {"S3_ENDPOINT": "http://minio:9000", "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo",
+           "S3_ACCESS_KEY": "test-user", "S3_SECRET_KEY": secret, "FEDEO_VERSION": "3.3.0",
+           "DATABASE_URL": "postgres://fedeo:test-db@db:5432/fedeo", "FEDEO_FILE_BACKEND": "s3"}
+    idle = {"image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
+            "restart": "unless-stopped", "networks": ["internal"],
+            "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(0)"], "interval": "1s"}}
+    import copy
+    services = {n: copy.deepcopy(idle) for n in ("backend", "frontend", "traefik", "node-exporter", "createbuckets")}
+    services["createbuckets"].update(command=["node", "-e", "process.exit(0)"], restart="no")
+    services["backend"]["environment"] = env
+    services["backend"]["image"] = "fedeo-storage-tools:pilot"
+    services["backend"]["entrypoint"] = ["node"]
+    services["backend"]["command"] = ["-e", "setInterval(()=>{},1000)"]
+    services["backend"]["depends_on"] = {"minio": {"condition": "service_healthy"}, "db": {"condition": "service_healthy"},
+                                                 "createbuckets": {"condition": "service_completed_successfully"}}
+    services["minio"] = {"image": "fedeo-minio-fixture:2025-09-07", "restart": "unless-stopped",
+        "command": ["server", "/data", "--console-address", ":9001"], "networks": ["internal"],
+        "volumes": [str(root / "minio") + ":/data"],
+        "environment": {"MINIO_ROOT_USER": "test-user", "MINIO_ROOT_PASSWORD": secret},
+        "healthcheck": {"test": ["CMD", "node", "-e", "fetch('http://127.0.0.1:9000/minio/health/live').then(r=>process.exit(r.ok?0:1))"], "interval": "1s"}}
+    services["db"] = {"image": "postgres:16-alpine", "restart": "unless-stopped", "networks": ["internal"],
+        "volumes": [str(root / "postgres") + ":/var/lib/postgresql/data"],
+        "environment": {"POSTGRES_USER": "fedeo", "POSTGRES_PASSWORD": "test-db", "POSTGRES_DB": "fedeo"},
+        "healthcheck": {"test": ["CMD", "pg_isready", "-U", "fedeo"], "interval": "1s"}}
+    config = {"name": project, "services": services, "networks": {"internal": {"name": project + "-internal"}}}
+    (root / "docker-compose.yml").write_text(json.dumps(m.literal(config)))
+    return root, project
+
+
+def seed(root, project, hidden=False):
+    code = """import { client, call } from './s3.mjs';
+const s=client({endpoint:'http://minio:9000',region:'eu-central-1',credentials:{accessKeyId:'test-user',secretAccessKey:'test-with-$-interpolation'}});
+await call(s,'CreateBucket',{Bucket:'fedeo'});
+await call(s,'PutObject',{Bucket:'fedeo',Key:'tenant/filesbyid/1/ä + % #.pdf',Body:Buffer.from('fixture'),ContentType:'application/pdf'});"""
+    if hidden:
+        code += "for(const Key of ['hidden','hidden/child']) await call(s,'PutObject',{Bucket:'fedeo',Key,Body:'fixture'});"
+    code += "s.destroy();"
+    docker("run", "--rm", "--pull", "never", "--network", project + "-internal", "--entrypoint", "node", "fedeo-storage-tools:pilot", "--input-type=module", "-e", code)
+
+
+def run(case):
+    root, project = fixture()
+    base = ["compose", "-p", project, "-f", str(root / "docker-compose.yml")]
+    try:
+        docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
+        seed(root, project, hidden=case == "hidden-prefix")
+        migration = m.Migration(root, "fedeo-storage-tools:pilot")
+        with migration.engine.lock():
+            if case == "hidden-prefix":
+                try:
+                    migration.plan()
+                    raise AssertionError("hidden key conflict must block planning")
+                except m.Error as error:
+                    assert "konflikt" in str(error)
+                assert not (root / ".updates/status.json").exists()
+                print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
+                return
+            migration.plan()
+            assert migration.state["count"] == 1
+            migration.migrate(True)
+            assert migration.state["phase"] == "verified"
+            assert m.u.read_json(root / ".updates/status.json")["recoveryRequired"]
+            # Simulate restart by reconstructing the process and persisted state.
+            migration = m.Migration(root)
+            if case == "rollback":
+                migration.rollback()
+                assert migration.state["phase"] == "rolled_back"
+                assert not m.u.read_json(root / ".updates/status.json").get("recoveryRequired")
+                assert migration.inspect(migration.state["containers"]["minio"]["id"])["State"]["Running"]
+            else:
+                migration.activate()
+                assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
+                assert migration.state["restoreVerifiedAt"]
+                old = migration.inspect(migration.state["containers"]["minio"]["id"])
+                assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
+                assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
+                try:
+                    migration.rollback()
+                    raise AssertionError("rollback after writes must fail")
+                except m.Error:
+                    pass
+        print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
+    except BaseException:
+        print("Failure evidence: " + str(root), flush=True)
+        log = root / ".updates/storage-migration/commands.log"
+        if log.exists():
+            print(log.read_text()[-8000:], flush=True)
+        ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project,
+                     "--filter", "label=com.docker.compose.service=objectstore").split()
+        for cid in ids:
+            logs = subprocess.run(["docker", "logs", "--tail", "60", cid], capture_output=True, text=True)
+            print(logs.stdout + logs.stderr, flush=True)
+        raise
+    finally:
+        ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split()
+        if ids:
+            docker("rm", "-f", *ids)
+        remaining = docker("ps", "-aq", "--filter", "label=io.fedeo.storage-job=" + str(root / ".updates/storage-migration")).split()
+        if remaining:
+            docker("rm", "-f", *remaining)
+        docker("network", "rm", project + "-internal")
+
+
+if __name__ == "__main__":
+    for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
+        assert case in ("hidden-prefix", "rollback", "activate")
+        run(case)