fix(storage): preserve preexisting Matrix health failures during restart

scripts/storage/tests/host_integration.py

@@ -9,6 +9,7 @@ import tempfile
 import tarfile
 import uuid
 import sys
+import time
 
 ROOT = Path(__file__).resolve().parents[2]
 spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
@@ -73,6 +74,21 @@ def run(case):
     try:
         docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
         seed(root, project, hidden=case == "hidden-prefix")
+        if case == "unhealthy-matrix":
+            config = json.loads((root / "docker-compose.yml").read_text())
+            config["services"]["matrix-element"] = {
+                "image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
+                "restart": "unless-stopped", "networks": ["internal"],
+                "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(1)"], "interval": "1s", "retries": 1}}
+            (root / "docker-compose.yml").write_text(json.dumps(config))
+            docker(*base, "up", "-d", "--pull", "never", "--no-deps", "matrix-element")
+            cid = docker(*base, "ps", "--quiet", "matrix-element")
+            for attempt in range(30):
+                if json.loads(docker("inspect", cid))[0]["State"]["Health"]["Status"] == "unhealthy":
+                    break
+                time.sleep(1)
+            else:
+                raise AssertionError("fixture must begin with an unhealthy Matrix container")
         retained = []
         if case.startswith("retired-"):
             (root / "matrix/postgres").mkdir(parents=True)
@@ -113,6 +129,10 @@ def run(case):
                 migration.activate()
                 assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
                 assert migration.state["restoreVerifiedAt"]
+                if case == "unhealthy-matrix":
+                    assert migration.state["preexistingUnhealthyServices"] == ["matrix-element"]
+                    cid = docker(*base, "ps", "--quiet", "matrix-element")
+                    assert migration.inspect(cid)["State"]["Running"]
                 old = migration.inspect(migration.state["containers"]["minio"]["id"])
                 assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
                 assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
@@ -151,5 +171,5 @@ def run(case):
 
 if __name__ == "__main__":
     for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
-        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate")
+        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate", "unhealthy-matrix")
         run(case)