#!/usr/bin/env python3 """Real Docker host-state test with synthetic services, never a real FEDEO instance.""" import importlib.util import json import os from pathlib import Path import subprocess import tempfile import tarfile import uuid import sys import time ROOT = Path(__file__).resolve().parents[2] spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py") m = importlib.util.module_from_spec(spec) spec.loader.exec_module(m) def docker(*args): return subprocess.check_output(["docker", *map(str, args)], text=True, stderr=subprocess.PIPE).strip() def fixture(): root = Path(tempfile.mkdtemp(prefix="fedeo-host-migration-")) project = "fedeo-pilot-" + uuid.uuid4().hex[:8] (root / "minio").mkdir() (root / "postgres").mkdir() (root / ".env").write_text("# synthetic test only\n") secret = "test-with-$-interpolation" env = {"S3_ENDPOINT": "http://minio:9000", "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo", "S3_ACCESS_KEY": "test-user", "S3_SECRET_KEY": secret, "FEDEO_VERSION": "3.3.0", "DATABASE_URL": "postgres://fedeo:test-db@db:5432/fedeo", "FEDEO_FILE_BACKEND": "s3"} idle = {"image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"], "restart": "unless-stopped", "networks": ["internal"], "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(0)"], "interval": "1s"}} import copy services = {n: copy.deepcopy(idle) for n in ("backend", "frontend", "traefik", "node-exporter", "createbuckets")} services["createbuckets"].update(command=["node", "-e", "process.exit(0)"], restart="no") services["backend"]["environment"] = env services["backend"]["image"] = "fedeo-storage-tools:pilot" services["backend"]["entrypoint"] = ["node"] services["backend"]["command"] = ["-e", "setInterval(()=>{},1000)"] services["backend"]["depends_on"] = {"minio": {"condition": "service_healthy"}, "db": {"condition": "service_healthy"}, "createbuckets": {"condition": "service_completed_successfully"}} services["minio"] = {"image": "fedeo-minio-fixture:2025-09-07", "restart": "unless-stopped", "command": ["server", "/data", "--console-address", ":9001"], "networks": ["internal"], "volumes": [str(root / "minio") + ":/data"], "environment": {"MINIO_ROOT_USER": "test-user", "MINIO_ROOT_PASSWORD": secret}, "healthcheck": {"test": ["CMD", "node", "-e", "fetch('http://127.0.0.1:9000/minio/health/live').then(r=>process.exit(r.ok?0:1))"], "interval": "1s"}} services["db"] = {"image": "postgres:16-alpine", "restart": "unless-stopped", "networks": ["internal"], "volumes": [str(root / "postgres") + ":/var/lib/postgresql/data"], "environment": {"POSTGRES_USER": "fedeo", "POSTGRES_PASSWORD": "test-db", "POSTGRES_DB": "fedeo"}, "healthcheck": {"test": ["CMD", "pg_isready", "-U", "fedeo"], "interval": "1s"}} config = {"name": project, "services": services, "networks": {"internal": {"name": project + "-internal"}}} (root / "docker-compose.yml").write_text(json.dumps(m.literal(config))) return root, project def seed(root, project, hidden=False): code = """import { client, call } from './s3.mjs'; const s=client({endpoint:'http://minio:9000',region:'eu-central-1',credentials:{accessKeyId:'test-user',secretAccessKey:'test-with-$-interpolation'}}); await call(s,'CreateBucket',{Bucket:'fedeo'}); await call(s,'PutObject',{Bucket:'fedeo',Key:'tenant/filesbyid/1/รค + % #.pdf',Body:Buffer.from('fixture'),ContentType:'application/pdf'});""" if hidden: code += "for(const Key of ['hidden','hidden/child']) await call(s,'PutObject',{Bucket:'fedeo',Key,Body:'fixture'});" code += "s.destroy();" docker("run", "--rm", "--pull", "never", "--network", project + "-internal", "--entrypoint", "node", "fedeo-storage-tools:pilot", "--input-type=module", "-e", code) def run(case): root, project = fixture() base = ["compose", "-p", project, "-f", str(root / "docker-compose.yml")] try: docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90") seed(root, project, hidden=case == "hidden-prefix") if case == "unhealthy-matrix": config = json.loads((root / "docker-compose.yml").read_text()) config["services"]["matrix-element"] = { "image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"], "restart": "unless-stopped", "networks": ["internal"], "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(1)"], "interval": "1s", "retries": 1}} (root / "docker-compose.yml").write_text(json.dumps(config)) docker(*base, "up", "-d", "--pull", "never", "--no-deps", "matrix-element") cid = docker(*base, "ps", "--quiet", "matrix-element") for attempt in range(30): if json.loads(docker("inspect", cid))[0]["State"]["Health"]["Status"] == "unhealthy": break time.sleep(1) else: raise AssertionError("fixture must begin with an unhealthy Matrix container") retained = [] if case.startswith("retired-"): (root / "matrix/postgres").mkdir(parents=True) (root / "matrix/postgres/retained.txt").write_text("retired matrix data") for service, mount in (("matrix-db", str(root / "matrix/postgres") + ":/var/lib/postgresql/data"), ("matrix-redis", "/data")): retained.append(docker("create", "--pull", "never", "--restart=no", "--network", project + "-internal", "--label", "com.docker.compose.project=" + project, "--label", "com.docker.compose.service=" + service, "-v", mount, "node:22-alpine", "node", "-e", "process.exit(0)")) migration = m.Migration(root, "fedeo-storage-tools:pilot") with migration.engine.lock(): if case == "hidden-prefix": try: migration.plan() raise AssertionError("hidden key conflict must block planning") except m.Error as error: assert "konflikt" in str(error) assert not (root / ".updates/status.json").exists() print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True) return migration.plan() assert migration.state["count"] == 1 if retained: assert migration.state["retainedServices"] == ["matrix-db", "matrix-redis"] assert migration.state["matrixPaths"] == ["matrix"] assert not set(migration.state["retainedServices"]) & m.u.read_json(migration.home / "target.json")["services"].keys() migration.migrate(True) assert migration.state["phase"] == "verified" assert m.u.read_json(root / ".updates/status.json")["recoveryRequired"] # Simulate restart by reconstructing the process and persisted state. migration = m.Migration(root) if case.endswith("rollback"): migration.rollback() assert migration.state["phase"] == "rolled_back" assert not m.u.read_json(root / ".updates/status.json").get("recoveryRequired") assert migration.inspect(migration.state["containers"]["minio"]["id"])["State"]["Running"] else: migration.activate() assert migration.state["phase"] == "complete" and migration.state["writesReleased"] assert migration.state["restoreVerifiedAt"] if case == "unhealthy-matrix": assert migration.state["preexistingUnhealthyServices"] == ["matrix-element"] cid = docker(*base, "ps", "--quiet", "matrix-element") assert migration.inspect(cid)["State"]["Running"] old = migration.inspect(migration.state["containers"]["minio"]["id"]) assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no" assert migration.state["network"] not in old["NetworkSettings"]["Networks"] try: migration.rollback() raise AssertionError("rollback after writes must fail") except m.Error: pass for cid in retained: old = migration.inspect(cid) assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no" if retained: with tarfile.open(migration.home / "backup/files.tar") as backup: assert backup.extractfile("matrix/postgres/retained.txt").read() == b"retired matrix data" print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True) except BaseException: print("Failure evidence: " + str(root), flush=True) log = root / ".updates/storage-migration/commands.log" if log.exists(): print(log.read_text()[-8000:], flush=True) ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project, "--filter", "label=com.docker.compose.service=objectstore").split() for cid in ids: logs = subprocess.run(["docker", "logs", "--tail", "60", cid], capture_output=True, text=True) print(logs.stdout + logs.stderr, flush=True) raise finally: ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split() if ids: docker("rm", "-f", "-v", *ids) remaining = docker("ps", "-aq", "--filter", "label=io.fedeo.storage-job=" + str(root / ".updates/storage-migration")).split() if remaining: docker("rm", "-f", *remaining) docker("network", "rm", project + "-internal") if __name__ == "__main__": for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]): assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate", "unhealthy-matrix") run(case)