flfeders/FEDEO
2
1
Fork 0
Code Issues 116 Pull Requests Actions Packages Projects 23 Releases 20 Wiki Activity

fix(updater): retain two verified backup generations

Browse Source
This commit is contained in:
Florian Federspiel
2026-10-02 15:27:07 +02:00
parent 23d49f7186
commit b01cf2c207
5 changed files with 434 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
scripts/selfhost-update.py
View File

@@ -17,6 +17,7 @@ from pathlib import Path
import re
import shutil
import signal
import stat
import subprocess
import sys
import time
@@ -34,6 +35,8 @@ LEGACY_MATRIX_SERVICES = {"matrix-db", "matrix-synapse", "matrix-redis", "matrix
"matrix-livekit", "matrix-rtc-jwt", "matrix-turn", "matrix-well-known"}
CORE_SERVICES = {"backend", "frontend", "db", "traefik", "node-exporter"}
IMAGE_ID = re.compile(r"sha256:[a-f0-9]{64}\Z")
UPDATE_JOB = re.compile(r"[0-9]{8}-[0-9]{6}-[a-f0-9]{8}\Z")
BACKUPS_TO_KEEP = 2
def compose_strings(value, *, escape=False):
@@ -882,6 +885,137 @@ class Updater:
raise
time.sleep(2)
def _backup_manifest(self, job, mounts, *, verify=False):
"""Only updater-owned regular backup files may be retained or removed."""
directory = job / "backup"
device = self.home.stat().st_dev
for path in (job, directory):
info = path.lstat()
if not stat.S_ISDIR(info.st_mode) or info.st_dev != device:
raise UpdateError("Backup-Verzeichnis ist kein reguläres lokales Verzeichnis.")
if any(path == directory or directory in path.parents or path in directory.parents for path in mounts):
raise UpdateError("Backup ist eingehängt oder wird von einem Container verwendet.")
files = set()
size = 0
for parent, directories, names in os.walk(directory, followlinks=False):
for name in directories + names:
path = Path(parent) / name
info = path.lstat()
if info.st_dev != device or not (stat.S_ISDIR(info.st_mode) or stat.S_ISREG(info.st_mode)):
raise UpdateError("Backup enthält Links, besondere Dateien oder fremde Dateisysteme.")
if stat.S_ISREG(info.st_mode):
files.add(str(path.relative_to(directory)))
size += info.st_blocks * 512
marker = job / "backup-pruned.json"
if marker.is_symlink() or (marker.exists() and not stat.S_ISREG(marker.lstat().st_mode)):
raise UpdateError("Backup-Löschprotokoll muss eine reguläre Datei sein.")
pending = read_json(marker, {})
if pending.get("status") == "deleting" and not verify:
# Resume an interrupted deletion using the manifest saved before removing any file.
manifest = pending["manifest"]
else:
manifest = read_json(directory / "complete.json")
checksums = manifest["sha256"]
required = {"database.dump", "globals.sql", "files.tar", "files.list", "containers.json",
"config/.env", "config/docker-compose.yml"}
if manifest.get("storageProvider") == "seaweedfs":
required |= {"storage-image.tar", "config/.updates/storage-provider.json"}
if (not isinstance(checksums, dict) or not required <= checksums.keys()
or manifest.get("database") != "database.dump" or manifest.get("files") != "files.tar"):
raise UpdateError("Unvollständiges Backup-Prüfmanifest.")
for name, digest in checksums.items():
path = Path(name)
if (path.is_absolute() or ".." in path.parts or str(path) != name or name == "complete.json"
or not re.fullmatch(r"[a-f0-9]{64}", str(digest))):
raise UpdateError("Ungültiger Pfad oder Prüfsumme im Backup-Manifest.")
if not files <= (checksums.keys() | {"complete.json"}):
raise UpdateError("Backup enthält unbekannte Dateien.")
if verify:
if files != checksums.keys() | {"complete.json"}:
raise UpdateError("Eine Datei der aufzubewahrenden Sicherung fehlt.")
for name, digest in checksums.items():
if file_sha(directory / name) != digest:
raise UpdateError("Prüfsumme einer aufzubewahrenden Sicherung stimmt nicht.")
return manifest, size
def _prune_backups(self, *, dry_run=False):
"""Caller holds the update lock. Never select migration or failed-job backups."""
jobs = []
for job in self.home.iterdir():
if not UPDATE_JOB.fullmatch(job.name) or job.is_symlink() or not job.is_dir():
continue
status_path = job / "status.json"
if status_path.is_symlink() or not status_path.is_file():
continue
state = read_json(job / "status.json", {})
if (state.get("command") == "apply" and state.get("status") == "succeeded"
and state.get("phase") == "complete" and state.get("recoveryRequired") is False
and state.get("maintenance") is False and state.get("job") == str(job)
and state.get("backup") == str(job / "backup")
and ((job / "backup").exists() or (job / "backup").is_symlink())):
started = datetime.datetime.fromisoformat(state["startedAt"])
if started.tzinfo is None:
raise UpdateError("Backup-Auftrag hat keine eindeutige Zeitangabe.")
jobs.append((started, job, state))
jobs.sort(key=lambda item: (item[0], item[1].name), reverse=True)
report = {"status": "preview" if dry_run else "complete", "keep": BACKUPS_TO_KEEP,
"checkedAt": now(), "retained": [], "removed": [], "planned": [], "skipped": []}
if not jobs:
return report
installed = read_json(self.home / "installed.json", {})
latest = jobs[0][2]
if (installed.get("manifest", {}).get("version") != latest.get("version")
or installed.get("manifest", {}).get("commit") != latest.get("revision")
or installed.get("composeSha256") != file_sha(self.root / "docker-compose.yml")):
raise UpdateError("Neueste erfolgreiche Sicherung gehört nicht zum installierten Stand; nichts gelöscht.")
# Include same-device bind mounts, which os.path.ismount alone cannot detect.
mounts = []
for line in Path("/proc/self/mountinfo").read_text().splitlines():
value = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), line.split()[4])
path = Path(value)
if path != Path("/") and (path == self.home or self.home in path.parents):
mounts.append(path)
for cid in self.runner.run(["docker", "ps", "--all", "--quiet"]).split():
for mount in self.inspect(cid).get("Mounts", []):
path = Path(mount["Source"]).resolve()
# A read-only host-root metrics mount does not consume backup data.
if path == Path("/") and not mount.get("RW", True):
continue
mounts.append(path)
# Check both retained generations before deleting any older generation.
for _, job, _ in jobs[:BACKUPS_TO_KEEP]:
self._backup_manifest(job, mounts, verify=True)
report["retained"].append(str(job / "backup"))
for _, job, _ in jobs[BACKUPS_TO_KEEP:]:
try:
manifest, size = self._backup_manifest(job, mounts)
item = {"backup": str(job / "backup"), "bytes": size}
if dry_run:
report["planned"].append(item)
continue
marker = {"status": "deleting", "startedAt": now(), "manifest": manifest,
"retained": report["retained"], **item}
write_json(job / "backup-pruned.json", marker)
shutil.rmtree(job / "backup")
write_json(job / "backup-pruned.json", {**marker, "status": "removed", "completedAt": now()})
report["removed"].append(item)
except (OSError, ValueError, KeyError, TypeError, UpdateError) as error:
report["skipped"].append({"job": str(job), "reason": str(error) if isinstance(error, UpdateError) else type(error).__name__})
if report["skipped"]:
report["status"] = "warning"
if not dry_run:
write_json(self.home / "backup-retention.json", report)
return report
def prune_backups(self, *, dry_run=False):
with self.lock():
state = read_json(self.home / "status.json", {})
if (state.get("status") in ("queued", "running") or state.get("recoveryRequired")
or state.get("maintenance") or state.get("phase") == "storage_migration"):
raise UpdateError("Aktiven oder unterbrochenen Update-/Migrationsauftrag zuerst abschließen.")
self.runner = self.runner_factory(self.home / "backup-retention.log")
return self._prune_backups(dry_run=dry_run)
def apply(self, accept_compose):
if not self.managed and not accept_compose:
raise UpdateError("Compose-Datei ist noch nicht verwaltet oder wurde bearbeitet. compose.diff prüfen, Anpassungen in die Override-Datei übertragen und --accept-compose-replacement verwenden.")
@@ -973,7 +1107,6 @@ class Updater:
self.save("checked", status="checked")
else:
self.apply(accept_compose)
return {**self.state, "plan": plan}
except BaseException as error:
if self.state.get("maintenance") and not self.state.get("mutationStarted"):
try:
@@ -989,6 +1122,25 @@ class Updater:
self.save(status="interrupted" if isinstance(error, (KeyboardInterrupt, SystemExit)) else "failed",
error=str(error) if isinstance(error, UpdateError) else type(error).__name__)
raise
# Retention is post-success maintenance, outside the update/recovery exception path.
# An I/O error or shutdown here must never stop the newly verified application.
if command == "apply":
try:
retention = self._prune_backups()
self.save(backupRetention=retention)
with self.runner.log.open("a") as log:
log.write(f"{now()} Backup-Aufbewahrung: {len(retention['retained'])} behalten, "
f"{len(retention['removed'])} entfernt, {len(retention['skipped'])} übersprungen.\n")
except Exception as error:
with contextlib.suppress(Exception):
warning = {"status": "warning", "checkedAt": now(), "keep": BACKUPS_TO_KEEP,
"error": str(error) if isinstance(error, UpdateError) else type(error).__name__}
write_json(self.home / "backup-retention.json", warning)
self.save(backupRetention=warning)
with self.runner.log.open("a") as log:
log.write(f"{now()} Backup-Bereinigung nicht abgeschlossen; Update erfolgreich. "
"Auf dem Host prune-backups --dry-run prüfen.\n")
return {**self.state, "plan": plan}
def resolve(self, version):
if not version or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.+-]{0,100}", version):
@@ -1073,13 +1225,16 @@ def main(argv=None):
print("Python ab Version 3.10 wird benötigt.", file=sys.stderr)
return 1
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("command", choices=("check", "apply", "status", "resolve", "adopt-storage"))
parser.add_argument("command", choices=("check", "apply", "status", "resolve", "adopt-storage", "prune-backups"))
parser.add_argument("--directory", default=os.environ.get("FEDEO_DIR", "/opt/fedeo"))
parser.add_argument("--version", help="Zielversion, z. B. 3.1.0; check wählt sonst den neuesten stabilen Release")
parser.add_argument("--accept-compose-replacement", action="store_true", help="Zuvor geprüften Austausch einer nicht verwalteten Compose-Datei akzeptieren")
parser.add_argument("--dry-run", action="store_true", help="Nur für prune-backups: Sicherungen prüfen und Löschvorschau ausgeben")
args = parser.parse_args(argv)
if args.command == "apply" and not args.version:
parser.error("apply benötigt eine explizite --version; zuerst check ausführen")
if args.dry_run and args.command != "prune-backups":
parser.error("--dry-run ist nur für prune-backups verfügbar")
os.umask(0o077)
signal.signal(signal.SIGTERM, lambda *_: sys.exit(143))
try:
@@ -1090,6 +1245,8 @@ def main(argv=None):
result = updater.resolve(args.version)
elif args.command == "adopt-storage":
result = updater.adopt_storage()
elif args.command == "prune-backups":
result = updater.prune_backups(dry_run=args.dry_run)
else:
result = updater.execute(args.command, args.version, args.accept_compose_replacement)
print(json.dumps(result, ensure_ascii=False, indent=2))