flfeders/FEDEO
2
1
Fork 0
Code Issues 116 Pull Requests Actions Packages Projects 23 Releases 20 Wiki Activity

fix(storage): preserve preexisting Matrix health failures during restart

Browse Source
This commit is contained in:
Florian Federspiel
2026-09-28 20:47:38 +02:00
parent 2b947c5e8f
commit 552a90e9e1
4 changed files with 78 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/minio-umstellung-pilot.md
View File

@@ -74,6 +74,13 @@ Bekannte lokale Matrix-Dienste werden erhalten, während der Kopie angehalten un
mit ihren Daten kalt gesichert. Andere Zusatzdienste/Compose-Overrides blockieren
den Pilot und benötigen einen angepassten Plan.
Der Plan hält auch vorhandene Container-Healthzustände fest. Bereits vor der
Migration als `unhealthy` bekannte Matrix-Nebendienste müssen beim Wiederanlauf
wieder laufen, blockieren aber nicht allein durch denselben Healthfehler die
FEDEO-Freigabe. FEDEO-Kerndienste und zuvor gesunde Matrix-Dienste müssen weiterhin
ihre Healthchecks bestehen; erforderliche Dienstabhängigkeiten bleiben erhalten.
Der Abschlusszustand nennt diese Altfehler unter `preexistingUnhealthyServices`.
Bereits durch ein früheres Update stillgelegte Matrix-Container dürfen weiterhin
zum Compose-Projekt gehören, obwohl sie nicht mehr in dessen Konfiguration stehen.
Der Pilot akzeptiert sie nur gestoppt und mit Neustartrichtlinie `no`, erhält ihre

15
scripts/selfhost-storage-migrate.py
View File

@@ -314,6 +314,7 @@ class Migration:
"matrixPaths": self.engine.matrix_backup_paths, "retainedServices": sorted(retained),
"containers": {name: {"id": c["Id"], "image": c["Image"],
"running": c["State"]["Running"], "restart": c["HostConfig"]["RestartPolicy"],
"health": c["State"].get("Health", {}).get("Status"),
"networks": c["NetworkSettings"]["Networks"]} for name, c in by_service.items()}}
# Planning errors must not leave a usable partial state file.
u.write_json(self.home / "original.json", config)
@@ -560,9 +561,19 @@ class Migration:
require(self.state.get("writesReleased"), "Schreibfreigabe fehlt.")
require(sha_file(self.root / "docker-compose.yml") == sha_file(self.home / "target.json"), "Aktive Compose-Datei verändert.")
names = [n for n, c in self.state["containers"].items() if c["running"] and n not in ("minio", "createbuckets", "traefik")]
self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "240", *names)
# Preserve Matrix without letting an already failing optional healthcheck
# cause a new FEDEO outage after cutover. Core services must always pass.
degraded = [n for n in names if n in u.LEGACY_MATRIX_SERVICES
and self.state["containers"][n].get("health") == "unhealthy"]
self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", *names)
self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never",
"--wait", "--wait-timeout", "240", *[n for n in names if n not in degraded])
for name in degraded:
cid = self.compose(self.root / "docker-compose.yml", "ps", "--all", "--quiet", name).strip()
require(cid and "\n" not in cid and self.inspect(cid)["State"]["Running"],
"Alter Matrix-Dienst wurde nicht wieder gestartet: " + name)
self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "traefik")
self.save("complete")
self.save("complete", preexistingUnhealthyServices=degraded)
def rollback(self):
require(self.state and not self.state.get("writesReleased"), "Nach Schreibfreigabe ist eine geprüfte Rückmigration erforderlich.")

22
scripts/storage/tests/host_integration.py
View File

@@ -9,6 +9,7 @@ import tempfile
import tarfile
import uuid
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
@@ -73,6 +74,21 @@ def run(case):
try:
docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
seed(root, project, hidden=case == "hidden-prefix")
if case == "unhealthy-matrix":
config = json.loads((root / "docker-compose.yml").read_text())
config["services"]["matrix-element"] = {
"image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
"restart": "unless-stopped", "networks": ["internal"],
"healthcheck": {"test": ["CMD", "node", "-e", "process.exit(1)"], "interval": "1s", "retries": 1}}
(root / "docker-compose.yml").write_text(json.dumps(config))
docker(*base, "up", "-d", "--pull", "never", "--no-deps", "matrix-element")
cid = docker(*base, "ps", "--quiet", "matrix-element")
for attempt in range(30):
if json.loads(docker("inspect", cid))[0]["State"]["Health"]["Status"] == "unhealthy":
break
time.sleep(1)
else:
raise AssertionError("fixture must begin with an unhealthy Matrix container")
retained = []
if case.startswith("retired-"):
(root / "matrix/postgres").mkdir(parents=True)
@@ -113,6 +129,10 @@ def run(case):
migration.activate()
assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
assert migration.state["restoreVerifiedAt"]
if case == "unhealthy-matrix":
assert migration.state["preexistingUnhealthyServices"] == ["matrix-element"]
cid = docker(*base, "ps", "--quiet", "matrix-element")
assert migration.inspect(cid)["State"]["Running"]
old = migration.inspect(migration.state["containers"]["minio"]["id"])
assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
@@ -151,5 +171,5 @@ def run(case):
if __name__ == "__main__":
for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate")
assert case in ("hidden-prefix", "rollback", "activate", "retired-rollback", "retired-activate", "unhealthy-matrix")
run(case)

37
scripts/storage/tests/test_host.py
View File

@@ -159,6 +159,43 @@ class HostSafetyTests(unittest.TestCase):
value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]}
self.assertEqual(m.resolved(m.literal(value)), value)
def restart_fixture(self):
migration = m.Migration(self.root)
(self.root / "docker-compose.yml").write_text('{}')
(migration.home / "target.json").write_text('{}')
migration.state = {"writesReleased": True, "phase": "released", "containers": {
"backend": {"running": True, "health": "unhealthy"},
"db": {"running": True, "health": "healthy"},
"matrix-synapse": {"running": True, "health": "healthy"},
"matrix-element": {"running": True, "health": "unhealthy"},
"matrix-redis": {"running": False, "health": None},
}}
migration.compose = Mock(return_value="matrix-element-id")
migration.inspect = Mock(return_value={"State": {"Running": True}})
return migration
def test_restart_preserves_existing_matrix_health_failure_but_waits_for_core(self):
migration = self.restart_fixture()
migration.start_application()
calls = [call.args for call in migration.compose.call_args_list]
start = next(args for args in calls if "up" in args and "--no-deps" not in args)
self.assertIn("matrix-element", start)
self.assertNotIn("matrix-redis", start)
wait = next(args for args in calls if "--wait" in args)
for name in ("backend", "db", "matrix-synapse"):
self.assertIn(name, wait)
self.assertNotIn("matrix-element", wait)
self.assertEqual(migration.state["preexistingUnhealthyServices"], ["matrix-element"])
self.assertEqual(migration.state["phase"], "complete")
def test_preexisting_unhealthy_matrix_must_still_restart(self):
migration = self.restart_fixture()
migration.inspect.return_value = {"State": {"Running": False}}
with self.assertRaisesRegex(m.Error, "nicht wieder gestartet"):
migration.start_application()
self.assertEqual(migration.state["phase"], "released")
self.assertFalse(any("traefik" in call.args for call in migration.compose.call_args_list))
if __name__ == "__main__":
unittest.main()