diff --git a/docs/minio-umstellung-pilot.md b/docs/minio-umstellung-pilot.md index a89c0ccb..430fb36e 100644 --- a/docs/minio-umstellung-pilot.md +++ b/docs/minio-umstellung-pilot.md @@ -74,6 +74,13 @@ Bekannte lokale Matrix-Dienste werden erhalten, während der Kopie angehalten un mit ihren Daten kalt gesichert. Andere Zusatzdienste/Compose-Overrides blockieren den Pilot und benötigen einen angepassten Plan. +Der Plan hält auch vorhandene Container-Healthzustände fest. Bereits vor der +Migration als `unhealthy` bekannte Matrix-Nebendienste müssen beim Wiederanlauf +wieder laufen, blockieren aber nicht allein durch denselben Healthfehler die +FEDEO-Freigabe. FEDEO-Kerndienste und zuvor gesunde Matrix-Dienste müssen weiterhin +ihre Healthchecks bestehen; erforderliche Dienstabhängigkeiten bleiben erhalten. +Der Abschlusszustand nennt diese Altfehler unter `preexistingUnhealthyServices`. + Bereits durch ein früheres Update stillgelegte Matrix-Container dürfen weiterhin zum Compose-Projekt gehören, obwohl sie nicht mehr in dessen Konfiguration stehen. Der Pilot akzeptiert sie nur gestoppt und mit Neustartrichtlinie `no`, erhält ihre diff --git a/scripts/selfhost-storage-migrate.py b/scripts/selfhost-storage-migrate.py index c529db7e..72f1496e 100644 --- a/scripts/selfhost-storage-migrate.py +++ b/scripts/selfhost-storage-migrate.py @@ -314,6 +314,7 @@ class Migration: "matrixPaths": self.engine.matrix_backup_paths, "retainedServices": sorted(retained), "containers": {name: {"id": c["Id"], "image": c["Image"], "running": c["State"]["Running"], "restart": c["HostConfig"]["RestartPolicy"], + "health": c["State"].get("Health", {}).get("Status"), "networks": c["NetworkSettings"]["Networks"]} for name, c in by_service.items()}} # Planning errors must not leave a usable partial state file. u.write_json(self.home / "original.json", config) @@ -560,9 +561,19 @@ class Migration: require(self.state.get("writesReleased"), "Schreibfreigabe fehlt.") require(sha_file(self.root / "docker-compose.yml") == sha_file(self.home / "target.json"), "Aktive Compose-Datei verändert.") names = [n for n, c in self.state["containers"].items() if c["running"] and n not in ("minio", "createbuckets", "traefik")] - self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "240", *names) + # Preserve Matrix without letting an already failing optional healthcheck + # cause a new FEDEO outage after cutover. Core services must always pass. + degraded = [n for n in names if n in u.LEGACY_MATRIX_SERVICES + and self.state["containers"][n].get("health") == "unhealthy"] + self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", *names) + self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", + "--wait", "--wait-timeout", "240", *[n for n in names if n not in degraded]) + for name in degraded: + cid = self.compose(self.root / "docker-compose.yml", "ps", "--all", "--quiet", name).strip() + require(cid and "\n" not in cid and self.inspect(cid)["State"]["Running"], + "Alter Matrix-Dienst wurde nicht wieder gestartet: " + name) self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "traefik") - self.save("complete") + self.save("complete", preexistingUnhealthyServices=degraded) def rollback(self): require(self.state and not self.state.get("writesReleased"), "Nach Schreibfreigabe ist eine geprüfte Rückmigration erforderlich.") diff --git a/scripts/storage/tests/host_integration.py b/scripts/storage/tests/host_integration.py index 07aaf45d..ee701421 100644 --- a/scripts/storage/tests/host_integration.py +++ b/scripts/storage/tests/host_integration.py @@ -9,6 +9,7 @@ import tempfile import tarfile import uuid import sys +import time ROOT = Path(__file__).resolve().parents[2] spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py") @@ -73,6 +74,21 @@ def run(case): try: docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90") seed(root, project, hidden=case == "hidden-prefix") + if case == "unhealthy-matrix": + config = json.loads((root / "docker-compose.yml").read_text()) + config["services"]["matrix-element"] = { + "image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"], + "restart": "unless-stopped", "networks": ["internal"], + "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(1)"], "interval": "1s", "retries": 1}} + (root / "docker-compose.yml").write_text(json.dumps(config)) + docker(*base, "up", "-d", "--pull", "never", "--no-deps", "matrix-element") + cid = docker(*base, "ps", "--quiet", "matrix-element") + for attempt in range(30): + if json.loads(docker("inspect", cid))[0]["State"]["Health"]["Status"] == "unhealthy": + break + time.sleep(1) + else: + raise AssertionError("fixture must begin with an unhealthy Matrix container") retained = [] if case.startswith("retired-"): (root / "matrix/postgres").mkdir(parents=True) @@ -113,6 +129,10 @@ def run(case): migration.activate() assert migration.state["phase"] == "complete" and migration.state["writesReleased"] assert migration.state["restoreVerifiedAt"] + if case == "unhealthy-matrix": + assert migration.state["preexistingUnhealthyServices"] == ["matrix-element"] + cid = docker(*base, "ps", "--quiet", "matrix-element") + assert migration.inspect(cid)["State"]["Running"] old = migration.inspect(migration.state["containers"]["minio"]["id"]) assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no" assert migration.state["network"] not in old["NetworkSettings"]["Networks"] @@ -151,5 +171,5 @@ def run(case): if __name__ == "__main__": for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]): - assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate") + assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate", "unhealthy-matrix") run(case) diff --git a/scripts/storage/tests/test_host.py b/scripts/storage/tests/test_host.py index 232241e0..d17eb019 100644 --- a/scripts/storage/tests/test_host.py +++ b/scripts/storage/tests/test_host.py @@ -159,6 +159,43 @@ class HostSafetyTests(unittest.TestCase): value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]} self.assertEqual(m.resolved(m.literal(value)), value) + def restart_fixture(self): + migration = m.Migration(self.root) + (self.root / "docker-compose.yml").write_text('{}') + (migration.home / "target.json").write_text('{}') + migration.state = {"writesReleased": True, "phase": "released", "containers": { + "backend": {"running": True, "health": "unhealthy"}, + "db": {"running": True, "health": "healthy"}, + "matrix-synapse": {"running": True, "health": "healthy"}, + "matrix-element": {"running": True, "health": "unhealthy"}, + "matrix-redis": {"running": False, "health": None}, + }} + migration.compose = Mock(return_value="matrix-element-id") + migration.inspect = Mock(return_value={"State": {"Running": True}}) + return migration + + def test_restart_preserves_existing_matrix_health_failure_but_waits_for_core(self): + migration = self.restart_fixture() + migration.start_application() + calls = [call.args for call in migration.compose.call_args_list] + start = next(args for args in calls if "up" in args and "--no-deps" not in args) + self.assertIn("matrix-element", start) + self.assertNotIn("matrix-redis", start) + wait = next(args for args in calls if "--wait" in args) + for name in ("backend", "db", "matrix-synapse"): + self.assertIn(name, wait) + self.assertNotIn("matrix-element", wait) + self.assertEqual(migration.state["preexistingUnhealthyServices"], ["matrix-element"]) + self.assertEqual(migration.state["phase"], "complete") + + def test_preexisting_unhealthy_matrix_must_still_restart(self): + migration = self.restart_fixture() + migration.inspect.return_value = {"State": {"Running": False}} + with self.assertRaisesRegex(m.Error, "nicht wieder gestartet"): + migration.start_application() + self.assertEqual(migration.state["phase"], "released") + self.assertFalse(any("traefik" in call.args for call in migration.compose.call_args_list)) + if __name__ == "__main__": unittest.main()