fix(storage): preserve preexisting Matrix health failures during restart

docs/minio-umstellung-pilot.md

@@ -74,6 +74,13 @@ Bekannte lokale Matrix-Dienste werden erhalten, während der Kopie angehalten un
 mit ihren Daten kalt gesichert. Andere Zusatzdienste/Compose-Overrides blockieren
 den Pilot und benötigen einen angepassten Plan.
 
+Der Plan hält auch vorhandene Container-Healthzustände fest. Bereits vor der
+Migration als `unhealthy` bekannte Matrix-Nebendienste müssen beim Wiederanlauf
+wieder laufen, blockieren aber nicht allein durch denselben Healthfehler die
+FEDEO-Freigabe. FEDEO-Kerndienste und zuvor gesunde Matrix-Dienste müssen weiterhin
+ihre Healthchecks bestehen; erforderliche Dienstabhängigkeiten bleiben erhalten.
+Der Abschlusszustand nennt diese Altfehler unter `preexistingUnhealthyServices`.
+
 Bereits durch ein früheres Update stillgelegte Matrix-Container dürfen weiterhin
 zum Compose-Projekt gehören, obwohl sie nicht mehr in dessen Konfiguration stehen.
 Der Pilot akzeptiert sie nur gestoppt und mit Neustartrichtlinie `no`, erhält ihre

scripts/selfhost-storage-migrate.py

@@ -314,6 +314,7 @@ class Migration:
                       "matrixPaths": self.engine.matrix_backup_paths, "retainedServices": sorted(retained),
                       "containers": {name: {"id": c["Id"], "image": c["Image"],
                           "running": c["State"]["Running"], "restart": c["HostConfig"]["RestartPolicy"],
+                          "health": c["State"].get("Health", {}).get("Status"),
                           "networks": c["NetworkSettings"]["Networks"]} for name, c in by_service.items()}}
         # Planning errors must not leave a usable partial state file.
         u.write_json(self.home / "original.json", config)
@@ -560,9 +561,19 @@ class Migration:
         require(self.state.get("writesReleased"), "Schreibfreigabe fehlt.")
         require(sha_file(self.root / "docker-compose.yml") == sha_file(self.home / "target.json"), "Aktive Compose-Datei verändert.")
         names = [n for n, c in self.state["containers"].items() if c["running"] and n not in ("minio", "createbuckets", "traefik")]
-        self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "240", *names)
+        # Preserve Matrix without letting an already failing optional healthcheck
+        # cause a new FEDEO outage after cutover. Core services must always pass.
+        degraded = [n for n in names if n in u.LEGACY_MATRIX_SERVICES
+                    and self.state["containers"][n].get("health") == "unhealthy"]
+        self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", *names)
+        self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never",
+                     "--wait", "--wait-timeout", "240", *[n for n in names if n not in degraded])
+        for name in degraded:
+            cid = self.compose(self.root / "docker-compose.yml", "ps", "--all", "--quiet", name).strip()
+            require(cid and "\n" not in cid and self.inspect(cid)["State"]["Running"],
+                    "Alter Matrix-Dienst wurde nicht wieder gestartet: " + name)
         self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "traefik")
-        self.save("complete")
+        self.save("complete", preexistingUnhealthyServices=degraded)
 
     def rollback(self):
         require(self.state and not self.state.get("writesReleased"), "Nach Schreibfreigabe ist eine geprüfte Rückmigration erforderlich.")

scripts/storage/tests/host_integration.py

@@ -9,6 +9,7 @@ import tempfile
 import tarfile
 import uuid
 import sys
+import time
 
 ROOT = Path(__file__).resolve().parents[2]
 spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
@@ -73,6 +74,21 @@ def run(case):
     try:
         docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
         seed(root, project, hidden=case == "hidden-prefix")
+        if case == "unhealthy-matrix":
+            config = json.loads((root / "docker-compose.yml").read_text())
+            config["services"]["matrix-element"] = {
+                "image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
+                "restart": "unless-stopped", "networks": ["internal"],
+                "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(1)"], "interval": "1s", "retries": 1}}
+            (root / "docker-compose.yml").write_text(json.dumps(config))
+            docker(*base, "up", "-d", "--pull", "never", "--no-deps", "matrix-element")
+            cid = docker(*base, "ps", "--quiet", "matrix-element")
+            for attempt in range(30):
+                if json.loads(docker("inspect", cid))[0]["State"]["Health"]["Status"] == "unhealthy":
+                    break
+                time.sleep(1)
+            else:
+                raise AssertionError("fixture must begin with an unhealthy Matrix container")
         retained = []
         if case.startswith("retired-"):
             (root / "matrix/postgres").mkdir(parents=True)
@@ -113,6 +129,10 @@ def run(case):
                 migration.activate()
                 assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
                 assert migration.state["restoreVerifiedAt"]
+                if case == "unhealthy-matrix":
+                    assert migration.state["preexistingUnhealthyServices"] == ["matrix-element"]
+                    cid = docker(*base, "ps", "--quiet", "matrix-element")
+                    assert migration.inspect(cid)["State"]["Running"]
                 old = migration.inspect(migration.state["containers"]["minio"]["id"])
                 assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
                 assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
@@ -151,5 +171,5 @@ def run(case):
 
 if __name__ == "__main__":
     for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
-        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate")
+        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate", "unhealthy-matrix")
         run(case)

scripts/storage/tests/test_host.py

@@ -159,6 +159,43 @@ class HostSafetyTests(unittest.TestCase):
         value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]}
         self.assertEqual(m.resolved(m.literal(value)), value)
 
+    def restart_fixture(self):
+        migration = m.Migration(self.root)
+        (self.root / "docker-compose.yml").write_text('{}')
+        (migration.home / "target.json").write_text('{}')
+        migration.state = {"writesReleased": True, "phase": "released", "containers": {
+            "backend": {"running": True, "health": "unhealthy"},
+            "db": {"running": True, "health": "healthy"},
+            "matrix-synapse": {"running": True, "health": "healthy"},
+            "matrix-element": {"running": True, "health": "unhealthy"},
+            "matrix-redis": {"running": False, "health": None},
+        }}
+        migration.compose = Mock(return_value="matrix-element-id")
+        migration.inspect = Mock(return_value={"State": {"Running": True}})
+        return migration
+
+    def test_restart_preserves_existing_matrix_health_failure_but_waits_for_core(self):
+        migration = self.restart_fixture()
+        migration.start_application()
+        calls = [call.args for call in migration.compose.call_args_list]
+        start = next(args for args in calls if "up" in args and "--no-deps" not in args)
+        self.assertIn("matrix-element", start)
+        self.assertNotIn("matrix-redis", start)
+        wait = next(args for args in calls if "--wait" in args)
+        for name in ("backend", "db", "matrix-synapse"):
+            self.assertIn(name, wait)
+        self.assertNotIn("matrix-element", wait)
+        self.assertEqual(migration.state["preexistingUnhealthyServices"], ["matrix-element"])
+        self.assertEqual(migration.state["phase"], "complete")
+
+    def test_preexisting_unhealthy_matrix_must_still_restart(self):
+        migration = self.restart_fixture()
+        migration.inspect.return_value = {"State": {"Running": False}}
+        with self.assertRaisesRegex(m.Error, "nicht wieder gestartet"):
+            migration.start_application()
+        self.assertEqual(migration.state["phase"], "released")
+        self.assertFalse(any("traefik" in call.args for call in migration.compose.call_args_list))
+
 
 if __name__ == "__main__":
     unittest.main()