fix(storage): preserve preexisting Matrix health failures during restart

scripts/storage/tests/host_integration.py

@@ -9,6 +9,7 @@ import tempfile
 import tarfile
 import uuid
 import sys
+import time
 
 ROOT = Path(__file__).resolve().parents[2]
 spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
@@ -73,6 +74,21 @@ def run(case):
     try:
         docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
         seed(root, project, hidden=case == "hidden-prefix")
+        if case == "unhealthy-matrix":
+            config = json.loads((root / "docker-compose.yml").read_text())
+            config["services"]["matrix-element"] = {
+                "image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
+                "restart": "unless-stopped", "networks": ["internal"],
+                "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(1)"], "interval": "1s", "retries": 1}}
+            (root / "docker-compose.yml").write_text(json.dumps(config))
+            docker(*base, "up", "-d", "--pull", "never", "--no-deps", "matrix-element")
+            cid = docker(*base, "ps", "--quiet", "matrix-element")
+            for attempt in range(30):
+                if json.loads(docker("inspect", cid))[0]["State"]["Health"]["Status"] == "unhealthy":
+                    break
+                time.sleep(1)
+            else:
+                raise AssertionError("fixture must begin with an unhealthy Matrix container")
         retained = []
         if case.startswith("retired-"):
             (root / "matrix/postgres").mkdir(parents=True)
@@ -113,6 +129,10 @@ def run(case):
                 migration.activate()
                 assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
                 assert migration.state["restoreVerifiedAt"]
+                if case == "unhealthy-matrix":
+                    assert migration.state["preexistingUnhealthyServices"] == ["matrix-element"]
+                    cid = docker(*base, "ps", "--quiet", "matrix-element")
+                    assert migration.inspect(cid)["State"]["Running"]
                 old = migration.inspect(migration.state["containers"]["minio"]["id"])
                 assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
                 assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
@@ -151,5 +171,5 @@ def run(case):
 
 if __name__ == "__main__":
     for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
-        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate")
+        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate", "unhealthy-matrix")
         run(case)

scripts/storage/tests/test_host.py

@@ -159,6 +159,43 @@ class HostSafetyTests(unittest.TestCase):
         value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]}
         self.assertEqual(m.resolved(m.literal(value)), value)
 
+    def restart_fixture(self):
+        migration = m.Migration(self.root)
+        (self.root / "docker-compose.yml").write_text('{}')
+        (migration.home / "target.json").write_text('{}')
+        migration.state = {"writesReleased": True, "phase": "released", "containers": {
+            "backend": {"running": True, "health": "unhealthy"},
+            "db": {"running": True, "health": "healthy"},
+            "matrix-synapse": {"running": True, "health": "healthy"},
+            "matrix-element": {"running": True, "health": "unhealthy"},
+            "matrix-redis": {"running": False, "health": None},
+        }}
+        migration.compose = Mock(return_value="matrix-element-id")
+        migration.inspect = Mock(return_value={"State": {"Running": True}})
+        return migration
+
+    def test_restart_preserves_existing_matrix_health_failure_but_waits_for_core(self):
+        migration = self.restart_fixture()
+        migration.start_application()
+        calls = [call.args for call in migration.compose.call_args_list]
+        start = next(args for args in calls if "up" in args and "--no-deps" not in args)
+        self.assertIn("matrix-element", start)
+        self.assertNotIn("matrix-redis", start)
+        wait = next(args for args in calls if "--wait" in args)
+        for name in ("backend", "db", "matrix-synapse"):
+            self.assertIn(name, wait)
+        self.assertNotIn("matrix-element", wait)
+        self.assertEqual(migration.state["preexistingUnhealthyServices"], ["matrix-element"])
+        self.assertEqual(migration.state["phase"], "complete")
+
+    def test_preexisting_unhealthy_matrix_must_still_restart(self):
+        migration = self.restart_fixture()
+        migration.inspect.return_value = {"State": {"Running": False}}
+        with self.assertRaisesRegex(m.Error, "nicht wieder gestartet"):
+            migration.start_application()
+        self.assertEqual(migration.state["phase"], "released")
+        self.assertFalse(any("traefik" in call.args for call in migration.compose.call_args_list))
+
 
 if __name__ == "__main__":
     unittest.main()