fix(storage): preserve retired Matrix containers during migration

scripts/storage/tests/host_integration.py

@@ -6,6 +6,7 @@ import os
 from pathlib import Path
 import subprocess
 import tempfile
+import tarfile
 import uuid
 import sys
 
@@ -72,6 +73,15 @@ def run(case):
     try:
         docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
         seed(root, project, hidden=case == "hidden-prefix")
+        retained = []
+        if case.startswith("retired-"):
+            (root / "matrix/postgres").mkdir(parents=True)
+            (root / "matrix/postgres/retained.txt").write_text("retired matrix data")
+            for service, mount in (("matrix-db", str(root / "matrix/postgres") + ":/var/lib/postgresql/data"),
+                                   ("matrix-redis", "/data")):
+                retained.append(docker("create", "--pull", "never", "--restart=no", "--network", project + "-internal",
+                    "--label", "com.docker.compose.project=" + project, "--label", "com.docker.compose.service=" + service,
+                    "-v", mount, "node:22-alpine", "node", "-e", "process.exit(0)"))
         migration = m.Migration(root, "fedeo-storage-tools:pilot")
         with migration.engine.lock():
             if case == "hidden-prefix":
@@ -85,12 +95,16 @@ def run(case):
                 return
             migration.plan()
             assert migration.state["count"] == 1
+            if retained:
+                assert migration.state["retainedServices"] == ["matrix-db", "matrix-redis"]
+                assert migration.state["matrixPaths"] == ["matrix"]
+                assert not set(migration.state["retainedServices"]) & m.u.read_json(migration.home / "target.json")["services"].keys()
             migration.migrate(True)
             assert migration.state["phase"] == "verified"
             assert m.u.read_json(root / ".updates/status.json")["recoveryRequired"]
             # Simulate restart by reconstructing the process and persisted state.
             migration = m.Migration(root)
-            if case == "rollback":
+            if case.endswith("rollback"):
                 migration.rollback()
                 assert migration.state["phase"] == "rolled_back"
                 assert not m.u.read_json(root / ".updates/status.json").get("recoveryRequired")
@@ -107,6 +121,12 @@ def run(case):
                     raise AssertionError("rollback after writes must fail")
                 except m.Error:
                     pass
+            for cid in retained:
+                old = migration.inspect(cid)
+                assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
+            if retained:
+                with tarfile.open(migration.home / "backup/files.tar") as backup:
+                    assert backup.extractfile("matrix/postgres/retained.txt").read() == b"retired matrix data"
         print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
     except BaseException:
         print("Failure evidence: " + str(root), flush=True)
@@ -122,7 +142,7 @@ def run(case):
     finally:
         ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split()
         if ids:
-            docker("rm", "-f", *ids)
+            docker("rm", "-f", "-v", *ids)
         remaining = docker("ps", "-aq", "--filter", "label=io.fedeo.storage-job=" + str(root / ".updates/storage-migration")).split()
         if remaining:
             docker("rm", "-f", *remaining)
@@ -131,5 +151,5 @@ def run(case):
 
 if __name__ == "__main__":
     for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
-        assert case in ("hidden-prefix", "rollback", "activate")
+        assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate")
         run(case)