feat(storage): add verified offline MinIO migration pilot

scripts/selfhost-storage-migrate.py

@@ -0,0 +1,607 @@
+#!/usr/bin/env python3
+"""Explicit, offline-capable pilot migration of an installer stack; Python 3.10+."""
+import argparse
+import copy
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import re
+import secrets
+import shutil
+import sys
+import time
+import urllib.parse
+
+HERE = Path(__file__).resolve().parent
+spec = importlib.util.spec_from_file_location("fedeo_update", HERE / "selfhost-update.py")
+u = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(u)
+Error = u.UpdateError
+GIB = 1024 ** 3
+
+
+class TransientStorageError(Error):
+    pass
+
+
+def require(condition, message):
+    if not condition:
+        raise Error(message)
+
+
+def sha_file(path):
+    h = hashlib.sha256()
+    with Path(path).open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def literal(value):
+    """Resolved Compose JSON still undergoes interpolation when used as a file."""
+    if isinstance(value, str):
+        return value.replace("$", "$$")
+    if isinstance(value, list):
+        return [literal(v) for v in value]
+    if isinstance(value, dict):
+        return {k: literal(v) for k, v in value.items()}
+    return value
+
+
+def resolved(value):
+    """Compose config serializes literal dollar signs escaped for another Compose read."""
+    if isinstance(value, str):
+        return value.replace("$$", "$")
+    if isinstance(value, list):
+        return [resolved(v) for v in value]
+    if isinstance(value, dict):
+        return {k: resolved(v) for k, v in value.items()}
+    return value
+
+
+def validate_minio_listing(root, bucket, inventory, *, exact=False):
+    """MinIO can hide a/b from LIST when a exists, while GET a/b still works.
+
+    Only inspect paths in the known single-drive layout; never decode or modify
+    xl.meta. A cold equality check guards the completeness of the S3 inventory.
+    """
+    source = root / "minio"
+    format_info = u.read_json(source / ".minio.sys/format.json", {})
+    require(format_info.get("format") == "xl-single" and format_info.get("xl", {}).get("version") == "3",
+            "MinIO-Datenformat ist für den zusätzlichen Vollständigkeitsabgleich nicht freigegeben.")
+    bucket_root = source / bucket
+    physical = {str(p.parent.relative_to(bucket_root)) for p in bucket_root.rglob("xl.meta") if p.is_file()}
+    for key in physical:
+        parts = key.split("/")
+        require(all("/".join(parts[:n]) not in physical for n in range(1, len(parts))),
+                "MinIO enthält einen Datei-/Verzeichniskonflikt; die S3-Auflistung kann Objekte verbergen.")
+    if exact:
+        require(physical == {o["Key"] for o in inventory["objects"]},
+                "Kalte MinIO-Metadaten und S3-Inventar weichen ab; keine vollständige Übernahme nachweisbar.")
+
+
+def validate_config(config, root):
+    services = config.get("services", {})
+    required = {"db", "minio", "createbuckets", "backend", "frontend", "traefik", "node-exporter"}
+    require(required <= services.keys() and not (services.keys() - required - u.LEGACY_MATRIX_SERVICES),
+            "Nur der Installer-Stack wird unterstützt; zusätzliche/fehlende Dienste zuerst klären.")
+    for service in services.values():
+        require(not any(service.get(k) for k in ("build", "profiles", "secrets", "configs", "volumes_from")),
+                "Builds, Profile und externe Konfigurationen benötigen einen eigenen Migrationsplan.")
+    store = services["minio"]
+    require(not store.get("ports") and set(store.get("networks", {})) == {"internal"},
+            "MinIO muss ausschließlich im internen Docker-Netz erreichbar sein.")
+    require(not services["db"].get("ports"), "Veröffentlichte Datenbankports benötigen einen gesonderten Schreiber-Nachweis.")
+    require(set(store.get("environment", {})) == {"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"},
+            "Zusätzliche MinIO-Konfiguration benötigt eine gesonderte Prüfung.")
+    for name, target, directory in (("minio", "/data", "minio"), ("db", "/var/lib/postgresql/data", "postgres")):
+        mounts = services[name].get("volumes", [])
+        require(len(mounts) == 1 and mounts[0].get("type") == "bind" and mounts[0].get("target") == target
+                and mounts[0].get("source") == str(root / directory), "Standard-Datenpfade fehlen: " + name)
+        require((root / directory).is_dir() and (root / directory).resolve() == root / directory,
+                "Datenpfad fehlt oder ist ein Link: " + directory)
+    env = services["backend"].get("environment", {})
+    require(env.get("FEDEO_FILE_BACKEND", "s3") == "s3" and not env.get("INFISICAL_CLIENT_ID"),
+            "Externe Speicher/Secret-Provider benötigen einen eigenen Adapter.")
+    require(env.get("S3_ENDPOINT", "").rstrip("/") == "http://minio:9000", "Abweichender S3-Endpunkt.")
+    require(all(env.get(k) for k in ("S3_BUCKET", "S3_REGION", "S3_ACCESS_KEY", "S3_SECRET_KEY")), "S3-Konfiguration unvollständig.")
+    require(re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", env["S3_BUCKET"]), "Ungültiger Bucketname.")
+    require(env["S3_ACCESS_KEY"] == store["environment"]["MINIO_ROOT_USER"] and
+            env["S3_SECRET_KEY"] == store["environment"]["MINIO_ROOT_PASSWORD"],
+            "Getrennte MinIO-Benutzer vorhanden; deren Rechte zunächst vollständig inventarisieren.")
+    db = urllib.parse.urlsplit(env.get("DATABASE_URL", ""))
+    db_env = services["db"].get("environment", {})
+    require(db.hostname == "db" and db.port in (None, 5432) and
+            urllib.parse.unquote(db.path.lstrip("/")) == db_env.get("POSTGRES_DB") and
+            urllib.parse.unquote(db.username or "") == db_env.get("POSTGRES_USER") and
+            urllib.parse.unquote(db.password or "") == db_env.get("POSTGRES_PASSWORD"),
+            "DATABASE_URL stimmt nicht mit der lokalen Sicherung überein.")
+    return env
+
+
+class Migration:
+    def __init__(self, root, tools_image=None):
+        os.umask(0o077)
+        self.engine = u.Updater(root)
+        self.root = self.engine.root
+        self.home = self.engine.home / "storage-migration"
+        require(not self.home.is_symlink(), "Migrationsverzeichnis darf kein Link sein.")
+        self.home.mkdir(mode=0o700, exist_ok=True)
+        self.runner = u.Runner(self.home / "commands.log")
+        self.state = u.read_json(self.home / "state.json", {})
+        self.tools_image = tools_image or self.state.get("toolsImage")
+        require(not self.state or tools_image in (None, self.state.get("toolsImage")), "Image eines bestehenden Auftrags darf nicht ersetzt werden.")
+        self.weed = u.read_json(HERE / "storage/seaweedfs.json")
+        self.helper_name = "fedeo-storage-helper-" + hashlib.sha256(str(self.root).encode()).hexdigest()[:12]
+        self.restore_name = self.helper_name + "-restore"
+
+    def docker(self, *args, **kwargs):
+        return self.runner.run(["docker", *args], **kwargs)
+
+    def compose(self, file, *args, **kwargs):
+        return self.docker("compose", "--project-directory", str(self.root), "--env-file", str(self.root / ".env"),
+                           "-p", self.state["project"], "-f", str(file), *args, **kwargs)
+
+    def save(self, phase=None, **fields):
+        self.state.update(fields, updatedAt=u.now())
+        if phase:
+            self.state["phase"] = phase
+        u.write_json(self.home / "state.json", self.state)
+
+    def guard(self):
+        # Compatible with the already installed 3.3 updater: it refuses recoveryRequired.
+        # Kept after cutover until a provider-aware application updater is installed.
+        u.write_json(self.engine.home / "status.json", {
+            "status": "failed", "phase": "storage_migration", "job": str(self.home),
+            "startedAt": self.state["startedAt"], "updatedAt": u.now(), "recoveryRequired": True,
+            "maintenance": not self.state.get("writesReleased", False), "mutationStarted": True,
+            "error": "Speichermigration: selfhost-storage-migrate.py status prüfen. Herkömmliche Updates sind gesperrt."})
+
+    def fingerprint(self):
+        return {name: sha_file(self.root / name) for name in ("docker-compose.yml", ".env")}
+
+    def inspect(self, container):
+        return json.loads(self.docker("inspect", container))[0]
+
+    def wait_ready(self, container, timeout=180):
+        for _ in range(timeout):
+            state = self.inspect(container)["State"]
+            require(state.get("Running"), "Ein benötigter Container wurde beendet.")
+            if state.get("Health", {}).get("Status") in (None, "healthy"):
+                return
+            time.sleep(1)
+        raise Error("Container wurde innerhalb der Wartezeit nicht bereit.")
+
+    def cleanup_helper(self):
+        ids = self.docker("ps", "-aq", "--filter", "name=^/" + self.helper_name + "$").split()
+        for container in ids:
+            require(self.inspect(container)["Config"].get("Labels", {}).get("io.fedeo.storage-job") == str(self.home),
+                    "Fremder Container belegt den Hilfsnamen.")
+            self.docker("rm", "-f", container)
+
+    def helper(self, *args, **kwargs):
+        deadline = time.monotonic() + 180
+        while True:
+            try:
+                return self.helper_once(*args, **kwargs)
+            except TransientStorageError:
+                if time.monotonic() >= deadline:
+                    raise
+                time.sleep(2)
+
+    def helper_once(self, command, *, source=False, admin=False, endpoint="http://objectstore:9000", directory="copy", backend_sdk=False):
+        self.cleanup_helper()
+        config = u.read_json(self.home / "credentials.json")
+        job = {"bucket": config["bucket"], "directory": "/work", "result": "/work/result.json"}
+        if command == "contract":
+            job["foreignBucket"] = "fedeo-denied-" + secrets.token_hex(8)
+        if source:
+            job["source"] = config["source"]
+        job["target"] = {**config["admin" if admin else "target"], "endpoint": endpoint}
+        u.write_json(self.home / "job.json", job)
+        work = self.home / directory
+        work.mkdir(mode=0o700, exist_ok=True)
+        (work / "error.json").unlink(missing_ok=True)
+        runtime = [self.tools_image, command]
+        if backend_sdk:
+            runtime = ["--entrypoint", "node", "-e", "FEDEO_STORAGE_SDK_ROOT=" + self.state["backendWorkingDir"],
+                       "-v", str(HERE / "storage") + ":/fedeo-check:ro", self.state["containers"]["backend"]["image"],
+                       "/fedeo-check/cli.mjs", command]
+        try:
+            self.docker("run", "--rm", "--pull", "never", "--name", self.helper_name,
+                        "--label", "io.fedeo.storage-job=" + str(self.home), "--network", self.state["network"],
+                        "--user", str(os.getuid()) + ":" + str(os.getgid()),
+                        "--read-only", "--cap-drop=ALL", "--security-opt", "no-new-privileges", "--tmpfs", "/tmp",
+                        "-v", str(self.home / "job.json") + ":/config/job.json:ro", "-v", str(work) + ":/work",
+                        *runtime, timeout=24 * 3600, output=self.home / "progress.jsonl")
+            return u.read_json(work / "result.json")
+        except Error:
+            detail = u.read_json(work / "error.json", {})
+            if detail.get("status") in (500, 502, 503, 504) or detail.get("code") in ("ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EHOSTUNREACH"):
+                raise TransientStorageError("S3 noch nicht bereit; Wiederholung blieb erfolglos.") from None
+            raise
+        finally:
+            self.cleanup_helper()
+
+    def plan(self):
+        require(not self.state, "Ein Plan existiert bereits; status prüfen oder vorhandenen Auftrag fortsetzen.")
+        require(not self.engine.status().get("recoveryRequired"), "Zuerst den offenen Update-Auftrag klären.")
+        require(self.tools_image, "--tools-image mit lokalem Hilfsimage angeben.")
+        require(all((self.root / n).is_file() and not (self.root / n).is_symlink() for n in (".env", "docker-compose.yml")),
+                "Reguläre .env und docker-compose.yml erforderlich.")
+        require(not any((self.root / n).exists() for n in ("docker-compose.override.yml", "docker-compose.override.yaml", "compose.yml", "compose.yaml")),
+                "Compose-Overrides zuerst in einem gesonderten Plan berücksichtigen.")
+        endpoint = self.docker("context", "inspect", "--format", "{{.Endpoints.docker.Host}}").strip()
+        require(os.environ.get("DOCKER_HOST", endpoint).startswith("unix://"),
+                "Nur ein lokaler Docker-Daemon wird unterstützt.")
+        info = json.loads(self.docker("info", "--format", "{{json .}}"))
+        require(info["OSType"] == "linux" and info["Architecture"] in ("amd64", "x86_64"), "Pilot benötigt Linux amd64.")
+        config = resolved(json.loads(self.docker("compose", "--project-directory", str(self.root), "--env-file", str(self.root / ".env"),
+                                       "-f", str(self.root / "docker-compose.yml"), "config", "--format", "json")))
+        env = validate_config(config, self.root)
+        # Reuse the updater's strict legacy Matrix path validation; the services remain intact.
+        self.engine.current = config
+        self.engine.retired_services = sorted(set(config["services"]) & u.LEGACY_MATRIX_SERVICES)
+        self.engine.matrix_backup_paths = []
+        self.engine.validate_legacy_matrix()
+        project = config["name"]
+        ids = self.docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split()
+        containers = [self.inspect(i) for i in ids]
+        by_service = {}
+        for c in containers:
+            labels = c["Config"].get("Labels", {})
+            name = labels.get("com.docker.compose.service")
+            require(name in config["services"] and name not in by_service and labels.get("com.docker.compose.oneoff", "False").lower() != "true",
+                    "Unbekannte oder doppelte Projektcontainer vorhanden.")
+            by_service[name] = c
+        require(all(by_service.get(n, {}).get("State", {}).get("Running") for n in ("db", "minio", "backend", "frontend", "traefik")),
+                "Die bestehende Instanz muss vor der Planung laufen.")
+        for name in ("minio", "backend", "db"):
+            actual = dict(v.split("=", 1) for v in by_service[name]["Config"]["Env"] if "=" in v)
+            require(all(actual.get(k) == str(v) for k, v in config["services"][name].get("environment", {}).items()),
+                    "Laufender Container und Compose-Umgebung weichen ab: " + name)
+        network = config["networks"]["internal"]["name"]
+        peers = json.loads(self.docker("network", "inspect", network))[0].get("Containers", {})
+        require(set(peers) <= {c["Id"] for c in containers}, "Weitere Container im internen Netzwerk; externe Schreiber zuerst klären.")
+        for name, target in (("minio", "/data"), ("db", "/var/lib/postgresql/data")):
+            mounts = [m for m in by_service[name]["Mounts"] if m["Destination"] == target]
+            require(len(mounts) == 1 and mounts[0]["Source"] == str(self.root / ("minio" if name == "minio" else "postgres")),
+                    "Laufender Datenmount weicht ab: " + name)
+        actual_store = by_service["minio"]
+        require(not actual_store["HostConfig"].get("PortBindings") and set(actual_store["NetworkSettings"]["Networks"]) == {network},
+                "Laufendes MinIO hat zusätzliche Netzwerkzugänge.")
+        require(len(actual_store["Mounts"]) == 1, "Laufendes MinIO hat zusätzliche Mounts.")
+        require(not any(p.is_symlink() for p in (self.root / "minio").rglob("*")), "Links in MinIO-Daten benötigen eigene Backup-Prüfung.")
+        tools = json.loads(self.docker("image", "inspect", self.tools_image))[0]
+        weed = json.loads(self.docker("image", "inspect", self.weed["image"]))[0]
+        self.tools_image = tools["Id"]
+        # Containerd may discard an untagged image index after a mutable build tag
+        # moves. Keep an independent immutable job reference throughout the pilot.
+        tools_tag = "fedeo-storage-tools:job-" + secrets.token_hex(12)
+        self.docker("image", "tag", self.tools_image, tools_tag)
+        self.state = {"schemaVersion": 1, "phase": "planning", "startedAt": u.now(), "project": project,
+                      "network": network, "toolsImage": self.tools_image, "weedImage": weed["Id"],
+                      "toolsRetainTag": tools_tag,
+                      "version": env.get("FEDEO_VERSION"), "fingerprint": self.fingerprint(), "writesReleased": False,
+                      "backendWorkingDir": by_service["backend"]["Config"].get("WorkingDir") or "/",
+                      "matrixPaths": self.engine.matrix_backup_paths, "containers": {name: {"id": c["Id"], "image": c["Image"],
+                          "running": c["State"]["Running"], "restart": c["HostConfig"]["RestartPolicy"],
+                          "networks": c["NetworkSettings"]["Networks"]} for name, c in by_service.items()}}
+        # Planning errors must not leave a usable partial state file.
+        u.write_json(self.home / "original.json", config)
+        for name in ("docker-compose.yml", ".env"):
+            u.atomic(self.home / ("original" + name.replace("/", "-")), (self.root / name).read_bytes())
+        u.write_json(self.home / "previous-update.json", u.read_json(self.engine.home / "status.json", {"status": "idle"}))
+        admin = {"accessKeyId": "fedeo-migration-" + secrets.token_hex(8), "secretAccessKey": secrets.token_urlsafe(36)}
+        app = {"accessKeyId": env["S3_ACCESS_KEY"], "secretAccessKey": env["S3_SECRET_KEY"]}
+        connection = {"region": env["S3_REGION"], "credentials": app}
+        u.write_json(self.home / "credentials.json", {"bucket": env["S3_BUCKET"],
+                     "source": {**connection, "endpoint": env["S3_ENDPOINT"]}, "target": connection,
+                     "admin": {**connection, "credentials": admin}})
+        inventory = self.helper("inventory", source=True, directory="preflight")
+        validate_minio_listing(self.root, env["S3_BUCKET"], inventory)
+        physical = int(self.runner.run(["du", "-sb", str(self.root / "minio")]).split()[0])
+        db_size = int(self.docker("exec", by_service["db"]["Id"], "sh", "-ec",
+            'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT pg_database_size(current_database())"').strip())
+        image_ids = sorted({c["Image"] for c in containers} | {tools["Id"], weed["Id"]})
+        image_bytes = sum(i["Size"] for i in json.loads(self.docker("image", "inspect", *image_ids)))
+        matrix_bytes = sum(int(self.runner.run(["du", "-sb", str(self.root / p)]).split()[0]) for p in self.state["matrixPaths"])
+        # Source backup + target + restored target + offline images + DB + spool + generous metadata reserve.
+        needed = physical + inventory["bytes"] * 3 + inventory["count"] * 256 * 1024 + db_size * 2 + image_bytes * 2 + matrix_bytes + 5 * GIB
+        require(shutil.disk_usage(self.root).free >= needed, f"Zu wenig freier Platz; mindestens {needed} Bytes erforderlich.")
+        self.state.update(imageIds=image_ids, requiredFreeBytes=needed, freeBytes=shutil.disk_usage(self.root).free,
+                          count=inventory["count"], bytes=inventory["bytes"])
+        require(not (self.root / "storage").exists(), "storage/ existiert bereits; kein leeres Migrationsziel.")
+        s3config = {"identities": [
+            {"name": "migration-admin", "credentials": [{"accessKey": admin["accessKeyId"], "secretKey": admin["secretAccessKey"]}],
+             "actions": ["Admin", "Read", "Write", "List", "Tagging"]},
+            {"name": "fedeo", "credentials": [{"accessKey": app["accessKeyId"], "secretKey": app["secretAccessKey"]}],
+             "actions": [action + ":" + env["S3_BUCKET"] for action in ("Read", "Write", "List")]}
+        ]}
+        u.write_json(self.home / "s3.json", s3config)
+        self.make_compose(config)
+        self.save("planned")
+
+    def make_compose(self, original):
+        target = copy.deepcopy(original)
+        store = copy.deepcopy(self.weed)
+        store["image"] = self.state["weedImage"]
+        store.update(container_name="fedeo-objectstore", restart="no", networks={"internal": {}},
+                     user=str(os.getuid()) + ":" + str(os.getgid()),
+                     environment={"FEDEO_S3_CONFIG_SHA256": sha_file(self.home / "s3.json")},
+                     volumes=[{"type": "bind", "source": str(self.root / "storage/seaweedfs"), "target": "/data"},
+                              {"type": "bind", "source": str(self.root / "storage/config"), "target": "/etc/fedeo", "read_only": True}])
+        stage = {"name": self.state["project"], "services": {"objectstore": store},
+                 "networks": {"internal": {"external": True, "name": self.state["network"]}}}
+        u.write_json(self.home / "stage.json", literal(stage))
+        del target["services"]["minio"]
+        del target["services"]["createbuckets"]
+        target["services"]["objectstore"] = copy.deepcopy(store)
+        target["services"]["objectstore"]["networks"]["internal"] = {"aliases": ["minio"]}
+        target["services"]["objectstore"]["restart"] = "unless-stopped"
+        dependencies = target["services"]["backend"].setdefault("depends_on", {})
+        dependencies.pop("minio", None)
+        dependencies.pop("createbuckets", None)
+        dependencies["objectstore"] = {"condition": "service_healthy"}
+        # Keep exactly the running application images; never couple this to an application/DB update.
+        for name, c in self.state["containers"].items():
+            if name in target["services"]:
+                target["services"][name]["image"] = c["image"]
+        target["services"]["backend"]["environment"]["FEDEO_RUN_MIGRATIONS"] = "false"
+        # SDK 3.962 streaming trailers leave aws-chunked in SeaweedFS object metadata.
+        # SigV4 remains active; the migration tool supplies an explicit SHA-256 checksum.
+        target["services"]["backend"]["environment"]["AWS_REQUEST_CHECKSUM_CALCULATION"] = "WHEN_REQUIRED"
+        u.write_json(self.home / "target.json", literal(target))
+        # Fail before maintenance if Compose cannot parse the generated configurations.
+        self.compose(self.home / "stage.json", "config", "--quiet")
+        self.compose(self.home / "target.json", "config", "--quiet")
+
+    def verify_original(self):
+        require(self.fingerprint() == self.state["fingerprint"], "Compose/.env wurde nach der Planung verändert.")
+        for name, c in self.state["containers"].items():
+            actual = self.inspect(c["id"])
+            require(actual["Image"] == c["image"], "Containeridentität verändert: " + name)
+
+    def freeze(self):
+        self.save("freezing")
+        self.guard()
+        for name, c in self.state["containers"].items():
+            if name != "db":
+                self.docker("update", "--restart=no", c["id"])
+        ids = [c["id"] for name, c in self.state["containers"].items() if name != "db"]
+        if ids:
+            self.docker("stop", "-t", "60", *ids)
+        self.assert_frozen()
+
+    def assert_frozen(self):
+        require(self.inspect(self.state["containers"]["minio"]["id"])["HostConfig"]["RestartPolicy"]["Name"] == "no",
+                "MinIO könnte automatisch wieder starten.")
+        for name, c in self.state["containers"].items():
+            if name in ("db", "minio"):
+                continue
+            actual = self.inspect(c["id"])
+            require(not actual["State"]["Running"] and actual["HostConfig"]["RestartPolicy"]["Name"] == "no",
+                    "Schreibpause wurde aufgehoben: " + name)
+        known = {c["id"] for c in self.state["containers"].values()}
+        peers = json.loads(self.docker("network", "inspect", self.state["network"]))[0].get("Containers", {})
+        for cid in set(peers) - known:
+            c = self.inspect(cid)
+            labels = c["Config"].get("Labels", {})
+            target = (labels.get("com.docker.compose.project") == self.state["project"] and
+                      labels.get("com.docker.compose.service") == "objectstore" and c["Image"] == self.state["weedImage"])
+            own_helper = labels.get("io.fedeo.storage-job") == str(self.home)
+            require(target or own_helper, "Ein zusätzlicher Container hat sich mit dem Speichernetz verbunden.")
+
+    def check_backup(self):
+        backup = self.home / "backup"
+        complete = u.read_json(backup / "complete.json", {})
+        require(complete.get("sha256"), "Abgeschlossene Sicherung fehlt.")
+        for name, expected in complete["sha256"].items():
+            require(Path(name).name == name and sha_file(backup / name) == expected,
+                    "Backup-Prüfsumme stimmt nicht: " + name)
+
+    def backup(self):
+        self.save("backing_up")
+        backup = self.home / "backup"
+        backup.mkdir(mode=0o700, exist_ok=True)
+        require(not self.inspect(self.state["containers"]["minio"]["id"])["State"]["Running"], "MinIO für kalte Sicherung stoppen.")
+        db = self.state["containers"]["db"]["id"]
+        self.docker("exec", db, "sh", "-ec", 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc', output=backup / "database.dump")
+        self.docker("exec", "-i", db, "pg_restore", "--list", input_file=backup / "database.dump", output=backup / "database.list")
+        self.docker("exec", db, "sh", "-ec", 'exec pg_dumpall -U "$POSTGRES_USER" --globals-only', output=backup / "globals.sql")
+        paths = ["minio", *self.state["matrixPaths"], "storage/config", ".env", "docker-compose.yml"]
+        if (self.root / "traefik/letsencrypt").is_dir():
+            paths.append("traefik/letsencrypt")
+        self.runner.run(["tar", "-cpf", str(backup / "files.tar"), "-C", str(self.root), *paths], timeout=7200)
+        self.docker("image", "save", "-o", str(backup / "images.tar"), *self.state["imageIds"], timeout=7200)
+        u.write_json(backup / "complete.json", {"at": u.now(), "sha256": {p.name: sha_file(p) for p in backup.iterdir() if p.is_file() and p.name != "complete.json"}})
+        self.save("backed_up")
+
+    def migrate(self, writers_exclusive):
+        require(writers_exclusive, "--writers-exclusive bestätigt: keine externen S3-/DB-Schreiber und kein paralleles manuelles Compose.")
+        require(self.state.get("phase") in ("planned", "freezing", "backing_up", "backed_up", "copying", "verifying", "verified"),
+                "Migration kann in dieser Phase nicht gestartet werden.")
+        self.cleanup_helper()
+        self.verify_original()
+        require(shutil.disk_usage(self.root).free >= (self.state["requiredFreeBytes"] if self.state["phase"] == "planned" else GIB),
+                "Nicht genug freier Speicher für Migration/Fortsetzung.")
+        if self.state["phase"] in ("planned", "freezing", "backing_up"):
+            (self.root / "storage/config").mkdir(parents=True, mode=0o700, exist_ok=True)
+            (self.root / "storage/seaweedfs/master").mkdir(parents=True, mode=0o700, exist_ok=True)
+            u.atomic(self.root / "storage/config/s3.json", (self.home / "s3.json").read_bytes())
+            u.atomic(self.root / "storage/config/start-seaweedfs.sh", (HERE / "storage/start-seaweedfs.sh").read_bytes())
+            self.freeze()
+            self.docker("start", self.state["containers"]["minio"]["id"])
+            self.wait_ready(self.state["containers"]["minio"]["id"])
+            current = self.helper("inventory", source=True, directory="frozen-inventory")
+            extra = max(0, current["bytes"] - self.state["bytes"]) * 4 + max(0, current["count"] - self.state["count"]) * 256 * 1024
+            require(shutil.disk_usage(self.root).free >= self.state["requiredFreeBytes"] + extra,
+                    "Bestand seit Planung gewachsen; Speicherreserve reicht nicht mehr.")
+            self.save(count=current["count"], bytes=current["bytes"])
+            self.docker("stop", "-t", "60", self.state["containers"]["minio"]["id"])
+            validate_minio_listing(self.root, u.read_json(self.home / "credentials.json")["bucket"], current, exact=True)
+            self.backup()
+        self.assert_frozen()
+        self.guard()
+        self.check_backup()
+        self.docker("start", self.state["containers"]["minio"]["id"])
+        self.wait_ready(self.state["containers"]["minio"]["id"])
+        self.compose(self.home / "stage.json", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "180")
+        self.helper("init", admin=True, directory="init")
+        self.helper("contract", directory="contract")
+        self.helper("contract", directory="backend-contract", backend_sdk=True)
+        self.save("copying")
+        self.helper("copy", source=True)
+        self.assert_frozen()
+        self.save("verifying")
+        self.helper("verify", source=True)
+        self.assert_frozen()
+        self.save("verified")
+
+    def restore_test(self):
+        # Start a separate container on newly extracted directories. Never count a
+        # successful archive listing or restart of the original directory as restore.
+        self.save("restore_testing")
+        restore = self.home / "restore-test"
+        if restore.exists():
+            # A previous interrupted restore is retained; use a fresh directory.
+            restore = self.home / ("restore-test-" + secrets.token_hex(4))
+        restore.mkdir(mode=0o700)
+        self.runner.run(["tar", "-xpf", str(self.home / "backup/seaweedfs.tar"), "-C", str(restore)], timeout=7200)
+        known = self.docker("ps", "-aq", "--filter", "name=^/" + self.restore_name + "$").split()
+        for cid in known:
+            require(self.inspect(cid)["Config"].get("Labels", {}).get("io.fedeo.storage-job") == str(self.home), "Fremder Restore-Container.")
+            self.docker("rm", "-f", cid)
+        target = u.read_json(self.home / "stage.json")["services"]["objectstore"]
+        try:
+            self.docker("run", "-d", "--pull", "never", "--name", self.restore_name,
+                        "--label", "io.fedeo.storage-job=" + str(self.home), "--network", self.state["network"],
+                        "--user", str(os.getuid()) + ":" + str(os.getgid()),
+                        "-v", str(restore / "storage/seaweedfs") + ":/data",
+                        "-v", str(restore / "storage/config") + ":/etc/fedeo:ro",
+                        "-e", "FEDEO_S3_CONFIG_SHA256=" + sha_file(self.home / "s3.json"),
+                        "--entrypoint", target["entrypoint"][0], self.state["weedImage"],
+                        *target["entrypoint"][1:], *target["command"])
+            self.helper("restore-verify", endpoint="http://" + self.restore_name + ":9000")
+            self.helper("contract", endpoint="http://" + self.restore_name + ":9000", directory="restore-contract")
+            self.save("cutover", restoreVerifiedAt=u.now(), seaweedBackupSha256=sha_file(self.home / "backup/seaweedfs.tar"))
+        finally:
+            self.docker("rm", "-f", self.restore_name)
+
+    def activate(self):
+        require(self.state.get("phase") in ("verified", "cutover", "restore_testing"), "Erst Kopie und Prüfung abschließen.")
+        require(not self.state.get("writesReleased"), "Schreibfreigabe ist bereits dauerhaft erfolgt.")
+        self.cleanup_helper()
+        self.assert_frozen()
+        # A killed contract test can leave only its journalled probe keys behind.
+        # Bring the target up without the legacy alias, remove those exact keys,
+        # then take the cold backup. Never delete unknown target objects.
+        self.compose(self.home / "stage.json", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "180")
+        for directory in ("contract", "backend-contract", "acceptance", "backend-acceptance"):
+            if (self.home / directory / "probe.json").is_file():
+                self.helper("cleanup-probe", directory=directory)
+        if self.state["phase"] == "verified":
+            self.verify_original()
+            self.helper("verify", source=True)
+            self.save("cutover")
+        self.guard()
+        self.check_backup()
+        old = self.state["containers"]["minio"]["id"]
+        self.docker("stop", "-t", "60", old)
+        validate_minio_listing(self.root, u.read_json(self.home / "credentials.json")["bucket"],
+                               u.read_json(self.home / "copy/inventory.json"), exact=True)
+        if self.state["network"] in self.inspect(old)["NetworkSettings"]["Networks"]:
+            self.docker("network", "disconnect", self.state["network"], old)
+        self.compose(self.home / "stage.json", "stop", "-t", "60", "objectstore")
+        # A cold backup is required before serving any business write on the new provider.
+        self.runner.run(["tar", "-cpf", str(self.home / "backup/seaweedfs.tar"), "-C", str(self.root), "storage"], timeout=7200)
+        self.restore_test()
+        u.atomic(self.root / "docker-compose.yml", (self.home / "target.json").read_bytes())
+        self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "--wait", "--wait-timeout", "180", "objectstore")
+        self.helper("restore-verify", endpoint="http://minio:9000")
+        self.helper("contract", endpoint="http://minio:9000", directory="acceptance")
+        self.helper("contract", endpoint="http://minio:9000", directory="backend-acceptance", backend_sdk=True)
+        # Persist before starting any backend/background process; no automatic rollback beyond here.
+        self.save("released", writesReleased=True, releasedAt=u.now(), provider="seaweedfs")
+        self.guard()
+        self.start_application()
+
+    def start_application(self):
+        require(self.state.get("writesReleased"), "Schreibfreigabe fehlt.")
+        require(sha_file(self.root / "docker-compose.yml") == sha_file(self.home / "target.json"), "Aktive Compose-Datei verändert.")
+        names = [n for n, c in self.state["containers"].items() if c["running"] and n not in ("minio", "createbuckets", "traefik")]
+        self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "240", *names)
+        self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "traefik")
+        self.save("complete")
+
+    def rollback(self):
+        require(self.state and not self.state.get("writesReleased"), "Nach Schreibfreigabe ist eine geprüfte Rückmigration erforderlich.")
+        require(sha_file(self.root / ".env") == self.state["fingerprint"][".env"], ".env verändert; Rückkehr manuell prüfen.")
+        require(sha_file(self.root / "docker-compose.yml") in {self.state["fingerprint"]["docker-compose.yml"], sha_file(self.home / "target.json")},
+                "Compose-Datei außerhalb der Migration verändert; Rückkehr manuell prüfen.")
+        self.cleanup_helper()
+        for cid in self.docker("ps", "-aq", "--filter", "name=^/" + self.restore_name + "$").split():
+            require(self.inspect(cid)["Config"].get("Labels", {}).get("io.fedeo.storage-job") == str(self.home), "Fremder Restore-Container.")
+            self.docker("rm", "-f", cid)
+        require(self.state["phase"] != "planning", "Unvollständiger Plan.")
+        self.save("rolling_back")
+        self.guard()
+        # May not exist when the failure occurred before the first target start.
+        ids = self.docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + self.state["project"],
+                          "--filter", "label=com.docker.compose.service=objectstore").split()
+        for cid in ids:
+            self.docker("update", "--restart=no", cid)
+            self.docker("stop", "-t", "60", cid)
+            if self.state["network"] in self.inspect(cid)["NetworkSettings"]["Networks"]:
+                self.docker("network", "disconnect", self.state["network"], cid)
+        u.atomic(self.root / "docker-compose.yml", (self.home / "originaldocker-compose.yml").read_bytes())
+        for name, c in self.state["containers"].items():
+            actual = self.inspect(c["id"])
+            if name == "minio" and self.state["network"] not in actual["NetworkSettings"]["Networks"]:
+                aliases = c["networks"][self.state["network"]].get("Aliases") or ["minio"]
+                args = [arg for alias in aliases for arg in ("--alias", alias)]
+                self.docker("network", "connect", *args, self.state["network"], c["id"])
+            policy = c["restart"]["Name"]
+            if policy == "on-failure" and c["restart"].get("MaximumRetryCount"):
+                policy += ":" + str(c["restart"]["MaximumRetryCount"])
+            self.docker("update", "--restart=" + policy, c["id"])
+        for name in ["db", "minio", *[n for n in self.state["containers"] if n not in ("db", "minio", "traefik")], "traefik"]:
+            c = self.state["containers"].get(name)
+            if c and c["running"]:
+                self.docker("start", c["id"])
+                self.wait_ready(c["id"])
+        self.save("rolled_back")
+        u.write_json(self.engine.home / "status.json", u.read_json(self.home / "previous-update.json"))
+
+    def public_status(self):
+        return {k: self.state[k] for k in ("phase", "version", "count", "bytes", "requiredFreeBytes", "freeBytes",
+                                          "writesReleased", "releasedAt", "updatedAt") if k in self.state}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=["plan", "migrate", "activate", "resume-start", "rollback", "status"])
+    parser.add_argument("--directory", default="/opt/fedeo")
+    parser.add_argument("--tools-image")
+    parser.add_argument("--writers-exclusive", action="store_true")
+    args = parser.parse_args()
+    migration = Migration(args.directory, args.tools_image)
+    try:
+        if args.command == "status":
+            print(json.dumps(migration.public_status(), indent=2)); return
+        with migration.engine.lock():
+            if args.command == "plan": migration.plan()
+            elif args.command == "migrate": migration.migrate(args.writers_exclusive)
+            elif args.command == "activate": migration.activate()
+            elif args.command == "resume-start": migration.start_application()
+            elif args.command == "rollback": migration.rollback()
+            print(json.dumps(migration.public_status(), indent=2))
+    except (Error, OSError, ValueError, KeyboardInterrupt) as error:
+        print("Migration angehalten: " + str(error) + "\nGeschütztes Protokoll: " + str(migration.home / "commands.log"), file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/storage/.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+tests

scripts/storage/Dockerfile

@@ -0,0 +1,6 @@
+FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32
+WORKDIR /opt/fedeo-storage
+COPY package.json package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund
+COPY s3.mjs cli.mjs ./
+ENTRYPOINT ["node", "/opt/fedeo-storage/cli.mjs"]

scripts/storage/README.md

@@ -0,0 +1,44 @@
+# Speicherwerkzeug für den betreuten Pilotbetrieb
+
+AWS SDK `3.962.0` entspricht dem bei Entwicklung vorliegenden Backend-Lockfile.
+Der konkrete Ablauf auf einer Instanz steht in
+[`docs/minio-umstellung-pilot.md`](../../docs/minio-umstellung-pilot.md).
+
+Lokaler Test (Linux amd64, Docker, Node 22):
+
+```sh
+npm ci --prefix scripts/storage
+npm test --prefix scripts/storage
+docker pull chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
+docker build -f scripts/storage/tests/Dockerfile.minio -t fedeo-minio-fixture:2025-09-07 scripts/storage/tests
+docker build -t fedeo-storage-tools:pilot scripts/storage
+node scripts/storage/tests/integration.mjs
+python3 -B scripts/storage/tests/host_integration.py
+```
+
+Der S3-Test verwendet ausschließlich neu angelegte Testcontainer mit künstlichen
+Daten. Er prüft über 1.000 Objekte, Pagination, 256 MiB, leere Dateien,
+Sonderzeichen, Metadaten, SigV4, 24-Stunden-Links, Streaming, verbotene Zugriffe,
+beschädigte Zielobjekte, erneute Kopie und echten Restore in ein frisches
+Verzeichnis. Die gesicherten Testdaten bleiben unter `/tmp/fedeo-storage-*`
+zur Prüfung erhalten; die Testcontainer werden entfernt.
+
+Der Host-Test verwendet einen echten PostgreSQL-/MinIO-/SeaweedFS-Stack und
+künstliche Anwendungscontainer. Er ersetzt keine fachliche Abnahme von FEDEO
+(PDF, WebDAV, Exporte, Mandantenrechte) auf der konkreten Instanz.
+
+Die erste Übernahme ist auf 200.000 Objekte und 256 MiB pro Objekt begrenzt.
+Das größte Objekt wird auf Platte gespult; RAM-Verbrauch wächst nicht mit dessen
+Größe. Multipart-Übernahmen und Sonderkonfigurationen werden abgelehnt.
+
+SeaweedFS 4.47 bewahrt bei SDK-Streaming mit automatischen Prüfsummen teilweise
+`Content-Encoding: aws-chunked` auf. Der Kopierer setzt einen vorher berechneten
+SHA-256-Checksum-Header. Die Pilot-Anwendung erhält
+`AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED`; der Vertragstest prüft diese
+Einstellung mit leeren und nicht leeren Streams. SigV4 bleibt aktiv.
+
+SeaweedFS-Herkunft: [Release 4.47](https://github.com/seaweedfs/seaweedfs/releases/tag/4.47).
+Das isolierte MinIO-Testimage verwendet das offizielle, per SHA-256 verifizierte
+[Release-Binary](https://github.com/minio/minio/releases/tag/RELEASE.2025-09-07T16-13-09Z),
+da die früheren Containerreferenzen nicht mehr abrufbar sind. Es wird nicht im
+Pilotpaket mitgeliefert und nicht über Kundendaten gestartet.

scripts/storage/build-pilot.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+storage_repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+storage_output="${1:-$storage_repo/.release/storage-pilot}"
+test ! -e "$storage_output"
+mkdir -p "$storage_output/scripts/storage" "$storage_output/docs"
+storage_weed="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["image"])' "$storage_repo/scripts/storage/seaweedfs.json")"
+if ! docker image inspect "$storage_weed" >/dev/null 2>&1; then docker pull "$storage_weed"; fi
+docker build -t fedeo-storage-tools:pilot "$storage_repo/scripts/storage"
+storage_tools_id="$(docker image inspect fedeo-storage-tools:pilot --format '{{.Id}}')"
+storage_weed_id="$(docker image inspect "$storage_weed" --format '{{.Id}}')"
+storage_tools="fedeo-storage-tools:pilot-${storage_tools_id#sha256:}"
+storage_weed_local="fedeo-seaweedfs:pilot-${storage_weed_id#sha256:}"
+docker image tag "$storage_tools_id" "$storage_tools"
+docker image tag "$storage_weed_id" "$storage_weed_local"
+cp "$storage_repo/scripts/selfhost-storage-migrate.py" "$storage_repo/scripts/selfhost-update.py" "$storage_output/scripts/"
+cp "$storage_repo/scripts/storage/"{s3.mjs,cli.mjs,seaweedfs.json,start-seaweedfs.sh,package.json,package-lock.json} "$storage_output/scripts/storage/"
+cp "$storage_repo/scripts/storage/pilot.py" "$storage_output/pilot.py"
+cp "$storage_repo/docs/minio-umstellung-pilot.md" "$storage_output/docs/"
+docker image save -o "$storage_output/images.tar" "$storage_tools" "$storage_weed_local"
+python3 - "$storage_output" "$storage_tools" "$storage_weed_local" "$storage_weed" <<'PY'
+import hashlib,json,pathlib,sys
+root=pathlib.Path(sys.argv[1]).resolve()
+spec=root/'scripts/storage/seaweedfs.json'
+config=json.loads(spec.read_text()); config['image']=sys.argv[3]
+spec.write_text(json.dumps(config,indent=2)+'\n')
+checksums={}
+for p in sorted(root.rglob('*')):
+    if p.is_file():
+        h=hashlib.sha256()
+        with p.open('rb') as f:
+            for chunk in iter(lambda:f.read(1024*1024), b''): h.update(chunk)
+        checksums[str(p.relative_to(root))]=h.hexdigest()
+(root/'bundle.json').write_text(json.dumps({'schemaVersion':1,'toolsImage':sys.argv[2],
+    'seaweedImage':sys.argv[3],'upstreamSeaweedImage':sys.argv[4],'sha256':checksums},indent=2)+'\n')
+PY
+tar -czf "$storage_output.tar.gz" -C "$(dirname "$storage_output")" "$(basename "$storage_output")"
+(cd "$(dirname "$storage_output")" && sha256sum "$(basename "$storage_output").tar.gz" > "$(basename "$storage_output").tar.gz.sha256")
+printf 'Pilotpaket: %s.tar.gz\n' "$storage_output"

scripts/storage/cli.mjs

@@ -0,0 +1,39 @@
+import { readFile } from 'node:fs/promises';
+import { client, call, inventory, transfer, contract, verifyManifest, atomicJson, cleanupProbe } from './s3.mjs';
+
+// Secrets enter through a root-only mounted JSON file, never through argv or logs.
+const [command, configPath = '/config/job.json'] = process.argv.slice(2);
+if (command === 'contract') process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED';
+let source, target;
+try {
+  const config = JSON.parse(await readFile(configPath, 'utf8'));
+  source = config.source && client(config.source);
+  target = config.target && client(config.target);
+  const bucket = config.bucket;
+  let result;
+  if (command === 'inventory') result = await inventory(source, bucket);
+  else if (command === 'copy' || command === 'verify') {
+    result = await transfer(source, target, bucket, config.directory || '/work', { verifyOnly: command === 'verify' });
+    result = { ...result, objects: undefined };
+  } else if (command === 'init') {
+    try { await call(target, 'HeadBucket', { Bucket: bucket }); }
+    catch (error) {
+      if (error.$metadata?.httpStatusCode !== 404) throw error;
+      await call(target, 'CreateBucket', { Bucket: bucket });
+    }
+    result = { bucket, initialized: true };
+  } else if (command === 'contract') result = await contract(target, config.target, bucket, { ...config, probePath: '/work/probe.json' });
+  else if (command === 'cleanup-probe') { await cleanupProbe(target, bucket, '/work/probe.json'); result = { cleaned: true }; }
+  else if (command === 'restore-verify') {
+    result = await verifyManifest(target, bucket, JSON.parse(await readFile(config.manifest || '/work/verified.json', 'utf8')));
+  } else throw new Error('Unbekannter Befehl.');
+  await atomicJson(config.result || '/work/result.json', result);
+  console.log(JSON.stringify(command === 'inventory' ? { count: result.count, bytes: result.bytes } : result));
+} catch (error) {
+  // SDK error messages/URLs may contain identifiers; keep public output bounded.
+  const detail = { error: error.name, code: error.code, status: error.$metadata?.httpStatusCode,
+    message: error.$metadata ? 'S3-Prüfung fehlgeschlagen; Zugriff/Konfiguration prüfen.' : error.message };
+  try { await atomicJson('/work/error.json', detail); } catch { /* Preserve the original error. */ }
+  console.error(JSON.stringify(detail));
+  process.exitCode = 1;
+} finally { source?.destroy(); target?.destroy(); }

scripts/storage/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@fedeo/storage-migration",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "engines": { "node": ">=22" },
+  "scripts": { "test": "node --test tests/*.test.mjs" },
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.962.0",
+    "@aws-sdk/s3-request-presigner": "3.962.0"
+  }
+}

scripts/storage/pilot.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Entry point shipped inside a verified, self-contained storage pilot bundle."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+ROOT = Path(__file__).resolve().parent
+
+
+def checksum(path):
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for block in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=["load-images", "plan", "migrate", "activate", "resume-start", "rollback", "status"])
+    parser.add_argument("--directory", default="/opt/fedeo")
+    parser.add_argument("--writers-exclusive", action="store_true")
+    args = parser.parse_args()
+    manifest = json.loads((ROOT / "bundle.json").read_text())
+    for name, expected in manifest["sha256"].items():
+        # The large image archive is checked when loading, not on every status call.
+        if name == "images.tar" and args.command != "load-images":
+            continue
+        path = ROOT / name
+        if path.resolve().is_relative_to(ROOT) and checksum(path) == expected:
+            continue
+        raise SystemExit("Paket-Prüfsumme stimmt nicht: " + name)
+    if args.command == "load-images":
+        subprocess.run(["docker", "image", "load", "-i", str(ROOT / "images.tar")], check=True)
+        for image in (manifest["toolsImage"], manifest["seaweedImage"]):
+            subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL)
+        print("Offline-Images geprüft und geladen. Nächster Schritt: plan.")
+        return
+    command = [sys.executable, str(ROOT / "scripts/selfhost-storage-migrate.py"), args.command, "--directory", args.directory]
+    if args.command == "plan":
+        command += ["--tools-image", manifest["toolsImage"]]
+    if args.writers_exclusive:
+        command += ["--writers-exclusive"]
+    raise SystemExit(subprocess.run(command).returncode)
+
+
+if __name__ == "__main__":
+    main()

scripts/storage/s3.mjs

@@ -0,0 +1,283 @@
+import { createRequire } from 'node:module';
+import { createHash, randomUUID } from 'node:crypto';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { mkdir, open, readFile, rename, rm } from 'node:fs/promises';
+import { Transform } from 'node:stream';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { join } from 'node:path';
+
+// A pilot also runs this contract inside the exact, already installed backend
+// image. Its own SDK dependency tree must pass before the host can cut over.
+const requireSdk = createRequire(process.env.FEDEO_STORAGE_SDK_ROOT
+  ? join(process.env.FEDEO_STORAGE_SDK_ROOT, 'package.json') : import.meta.url);
+const S3 = requireSdk('@aws-sdk/client-s3');
+const { getSignedUrl } = requireSdk('@aws-sdk/s3-request-presigner');
+
+// The pilot deliberately has a tested bound; no untested multipart conversion.
+export const MAX_OBJECT_BYTES = 256 * 1024 * 1024;
+export const MAX_OBJECTS = 200_000;
+export const client = config => new S3.S3Client({ ...config, forcePathStyle: true, maxAttempts: 3 });
+export const call = (s3, operation, input) => s3.send(new S3[`${operation}Command`](input));
+export const canonical = value => JSON.stringify(value, (_, v) => v && typeof v === 'object' && !Array.isArray(v)
+  ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a.localeCompare(b))) : v);
+export const digest = value => createHash('sha256').update(canonical(value)).digest('hex');
+const requireThat = (condition, reason) => { if (!condition) throw new Error(reason); };
+
+export async function atomicJson(path, value) {
+  const temp = `${path}.tmp`;
+  const file = await open(temp, 'w', 0o600);
+  try { await file.writeFile(JSON.stringify(value, null, 2) + '\n'); await file.sync(); }
+  finally { await file.close(); }
+  await rename(temp, path);
+  const parent = await open(join(path, '..'), 'r');
+  try { await parent.sync(); } finally { await parent.close(); }
+}
+
+export function validateKeys(objects) {
+  requireThat(objects.length <= MAX_OBJECTS, `Pilotgrenze: höchstens ${MAX_OBJECTS} Objekte.`);
+  const keys = new Set(objects.map(o => o.Key));
+  requireThat(keys.size === objects.length, 'Doppelte Objektschlüssel.');
+  for (const { Key: key, Size: size } of objects) {
+    requireThat(typeof key === 'string' && key.length > 0 && Buffer.byteLength(key) <= 1024,
+      'Ungültiger Objektschlüssel.');
+    const segments = key.split('/');
+    requireThat(!/[\x00-\x1f\x7f\\]/.test(key) && segments.every(s => s && s !== '.' && s !== '..'),
+      `Nicht verlustfrei übertragbarer Schlüssel: ${JSON.stringify(key)}`);
+    requireThat(Number.isSafeInteger(size) && size >= 0 && size <= MAX_OBJECT_BYTES,
+      `Objekt außerhalb der getesteten Größe (256 MiB): ${JSON.stringify(key)}`);
+    for (let i = 1; i < segments.length; i++) {
+      requireThat(!keys.has(segments.slice(0, i).join('/')), `Datei-/Verzeichniskonflikt: ${JSON.stringify(key)}`);
+    }
+  }
+}
+
+export async function list(s3, Bucket) {
+  const result = [];
+  let ContinuationToken;
+  do {
+    const page = await call(s3, 'ListObjectsV2', { Bucket, ContinuationToken });
+    result.push(...(page.Contents || []).map(o => ({ Key: o.Key, Size: o.Size, ETag: o.ETag,
+      LastModified: o.LastModified?.toISOString() })));
+    requireThat(result.length <= MAX_OBJECTS, `Pilotgrenze: höchstens ${MAX_OBJECTS} Objekte.`);
+    requireThat(!page.IsTruncated || (page.NextContinuationToken && page.NextContinuationToken !== ContinuationToken),
+      'Ungültige S3-Paginierung.');
+    ContinuationToken = page.IsTruncated ? page.NextContinuationToken : undefined;
+  } while (ContinuationToken);
+  return result.sort((a, b) => Buffer.compare(Buffer.from(a.Key), Buffer.from(b.Key)));
+}
+
+async function optional(s3, operation, Bucket, absentCodes) {
+  try { return await call(s3, operation, { Bucket }); }
+  catch (error) { if (absentCodes.includes(error.name)) return {}; throw error; }
+}
+
+function privateAcl(acl) {
+  return (acl.Grants || []).every(g => g.Grantee?.Type === 'CanonicalUser' &&
+    (g.Grantee.ID || '') === (acl.Owner?.ID || '') && g.Permission === 'FULL_CONTROL');
+}
+
+export async function inspectBucket(s3, Bucket) {
+  const buckets = await call(s3, 'ListBuckets', {});
+  requireThat(buckets.Buckets?.length === 1 && buckets.Buckets[0].Name === Bucket,
+    'Die Quelle muss genau den FEDEO-Bucket enthalten; weitere Buckets separat klären.');
+  const v = await call(s3, 'GetBucketVersioning', { Bucket });
+  requireThat(!v.Status, 'Aktivierte oder ausgesetzte Versionierung wird nicht unterstützt.');
+  const versions = await call(s3, 'ListObjectVersions', { Bucket, MaxKeys: 1000 });
+  // Even for an unversioned bucket MinIO returns current versions with VersionId=null.
+  requireThat(!(versions.DeleteMarkers?.length) && (versions.Versions || []).every(x =>
+    x.IsLatest && x.VersionId === 'null'), 'Alte Versionen oder Delete-Marker vorhanden.');
+  // A versioning flag above is mandatory; still inspect all version pages to detect old history.
+  let page = versions;
+  while (page.IsTruncated) {
+    const next = await call(s3, 'ListObjectVersions', { Bucket, KeyMarker: page.NextKeyMarker,
+      VersionIdMarker: page.NextVersionIdMarker });
+    requireThat(!(next.DeleteMarkers?.length) && (next.Versions || []).every(x => x.IsLatest && x.VersionId === 'null'),
+      'Alte Versionen oder Delete-Marker vorhanden.');
+    requireThat(next.NextKeyMarker !== page.NextKeyMarker || !next.IsTruncated, 'Ungültige Versionspaginierung.');
+    page = next;
+  }
+  const lock = await optional(s3, 'GetObjectLockConfiguration', Bucket, ['ObjectLockConfigurationNotFoundError', 'ObjectLockConfigurationNotFound']);
+  requireThat(!lock.ObjectLockConfiguration?.ObjectLockEnabled, 'Object Lock wird nicht unterstützt.');
+  const encryption = await optional(s3, 'GetBucketEncryption', Bucket, ['ServerSideEncryptionConfigurationNotFoundError']);
+  requireThat(!encryption.ServerSideEncryptionConfiguration?.Rules?.length, 'Bucket-Verschlüsselung benötigt einen eigenen Adapter.');
+  const policy = await optional(s3, 'GetBucketPolicy', Bucket, ['NoSuchBucketPolicy']);
+  requireThat(!policy.Policy, 'Eigene Bucket-Policy benötigt eine gesonderte Übernahme.');
+  const lifecycle = await optional(s3, 'GetBucketLifecycleConfiguration', Bucket, ['NoSuchLifecycleConfiguration']);
+  requireThat(!lifecycle.Rules?.length, 'Lifecycle-Regeln vorhanden.');
+  const replication = await optional(s3, 'GetBucketReplication', Bucket, ['ReplicationConfigurationNotFoundError']);
+  requireThat(!replication.ReplicationConfiguration, 'Bucket-Replikation vorhanden.');
+  const notification = await call(s3, 'GetBucketNotificationConfiguration', { Bucket });
+  requireThat(!['TopicConfigurations', 'QueueConfigurations', 'LambdaFunctionConfigurations'].some(k => notification[k]?.length)
+    && !notification.EventBridgeConfiguration, 'Bucket-Benachrichtigungen vorhanden.');
+  const uploads = await call(s3, 'ListMultipartUploads', { Bucket, MaxUploads: 1 });
+  requireThat(!uploads.Uploads?.length, 'Unvollständige Multipart-Uploads vorhanden.');
+  const tags = await optional(s3, 'GetBucketTagging', Bucket, ['NoSuchTagSet', 'NoSuchTagSetError']);
+  requireThat(!tags.TagSet?.length, 'Bucket-Tags vorhanden.');
+  const cors = await optional(s3, 'GetBucketCors', Bucket, ['NoSuchCORSConfiguration']);
+  requireThat(!cors.CORSRules?.length, 'Eigene CORS-Konfiguration vorhanden.');
+  requireThat(privateAcl(await call(s3, 'GetBucketAcl', { Bucket })), 'Bucket-ACL ist nicht privat.');
+}
+
+export function metadata(head) {
+  requireThat(!head.ServerSideEncryption && !head.SSECustomerAlgorithm && !head.ObjectLockMode &&
+    !head.ObjectLockLegalHoldStatus && !head.WebsiteRedirectLocation && (!head.StorageClass || head.StorageClass === 'STANDARD'),
+  'Nicht unterstützte Objektattribute gefunden.');
+  const result = {};
+  for (const key of ['ContentType', 'ContentDisposition', 'ContentEncoding', 'ContentLanguage', 'CacheControl', 'Metadata']) {
+    if (head[key] !== undefined) result[key] = head[key];
+  }
+  result.Metadata ||= {};
+  if (head.Expires) result.Expires = head.Expires.toISOString();
+  return result;
+}
+
+export async function inventory(s3, Bucket) {
+  await inspectBucket(s3, Bucket);
+  const objects = await list(s3, Bucket);
+  validateKeys(objects);
+  for (const object of objects) {
+    const input = { Bucket, Key: object.Key };
+    const head = await call(s3, 'HeadObject', input);
+    requireThat(head.ETag === object.ETag && head.ContentLength === object.Size, 'Quelle während Inventarisierung verändert.');
+    object.metadata = metadata(head);
+    requireThat(!(await call(s3, 'GetObjectTagging', input)).TagSet?.length, `Objekt-Tags vorhanden: ${JSON.stringify(object.Key)}`);
+    requireThat(privateAcl(await call(s3, 'GetObjectAcl', input)), 'Objekt-ACL ist nicht privat.');
+  }
+  return { schemaVersion: 1, bucket: Bucket, count: objects.length,
+    bytes: objects.reduce((n, o) => n + o.Size, 0), objects };
+}
+
+export async function readHash(s3, Bucket, object, spool) {
+  const response = await call(s3, 'GetObject', { Bucket, Key: object.Key, ...(object.ETag ? { IfMatch: object.ETag } : {}) });
+  const hash = createHash('sha256');
+  let bytes = 0;
+  const inspect = new Transform({ transform(chunk, _, next) {
+    bytes += chunk.length;
+    if (bytes > MAX_OBJECT_BYTES) return next(new Error('Objekt überschreitet Größenlimit.'));
+    hash.update(chunk); next(null, chunk);
+  }});
+  if (spool) await pipeline(response.Body, inspect, createWriteStream(spool, { mode: 0o600 }));
+  else for await (const chunk of response.Body) { bytes += chunk.length; hash.update(chunk);
+    requireThat(bytes <= MAX_OBJECT_BYTES, 'Objekt überschreitet Größenlimit.'); }
+  requireThat(bytes === object.Size, 'Objektgröße während Lesen verändert.');
+  return { sha256: hash.digest('hex'), size: bytes, metadata: metadata(response) };
+}
+
+export async function transfer(source, target, Bucket, directory, { verifyOnly = false } = {}) {
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+  const current = await inventory(source, Bucket);
+  const manifestPath = join(directory, 'inventory.json');
+  let previous;
+  try { previous = JSON.parse(await readFile(manifestPath, 'utf8')); }
+  catch (error) { if (error.code !== 'ENOENT') throw error; }
+  requireThat(!previous || canonical(previous) === canonical(current), 'Quellinventar seit Beginn verändert.');
+  const targetObjects = await list(target, Bucket);
+  requireThat(previous || targetObjects.length === 0, 'Ziel ist bei erster Kopie nicht leer.');
+  const sourceKeys = new Set(current.objects.map(o => o.Key));
+  requireThat(targetObjects.every(o => sourceKeys.has(o.Key)), 'Unerklärte zusätzliche Zielobjekte vorhanden.');
+  if (!previous) await atomicJson(manifestPath, current);
+  const existing = new Map(targetObjects.map(o => [o.Key, o]));
+  const journal = await open(join(directory, 'objects.jsonl'), 'a', 0o600);
+  const hashes = [];
+  try {
+    for (const object of current.objects) {
+      const spool = join(directory, 'object.spool');
+      try {
+        const original = await readHash(source, Bucket, object, verifyOnly ? undefined : spool);
+        requireThat(canonical(original.metadata) === canonical(object.metadata), 'Quellmetadaten verändert.');
+        let copied;
+        if (existing.has(object.Key) && existing.get(object.Key).Size === object.Size) {
+          copied = await readHash(target, Bucket, { Key: object.Key, Size: object.Size });
+        }
+        if (canonical(copied) !== canonical(original)) {
+          requireThat(!verifyOnly, `Integritätsfehler: ${JSON.stringify(object.Key)}`);
+          const meta = { ...object.metadata };
+          if (meta.Expires) meta.Expires = new Date(meta.Expires);
+          await call(target, 'PutObject', { Bucket, Key: object.Key, ...meta,
+            ContentLength: object.Size, ChecksumSHA256: Buffer.from(original.sha256, 'hex').toString('base64'),
+            Body: object.Size === 0 ? Buffer.alloc(0) : createReadStream(spool) });
+          copied = await readHash(target, Bucket, { Key: object.Key, Size: object.Size });
+          requireThat(canonical(copied) === canonical(original), `Integritätsfehler nach Kopie: ${JSON.stringify(object.Key)}`);
+        }
+        hashes.push({ key: object.Key, ...original });
+        await journal.writeFile(JSON.stringify({ key: object.Key, ...original, verifiedAt: new Date().toISOString() }) + '\n');
+        await journal.sync();
+        process.stdout.write(JSON.stringify({ phase: verifyOnly ? 'verifying' : 'copying', completed: hashes.length, total: current.count }) + '\n');
+      } finally { await rm(spool, { force: true }); }
+    }
+  } finally { await journal.close(); }
+  const after = await inventory(source, Bucket);
+  requireThat(canonical(after) === canonical(current), 'Quelle während Kopie/Prüfung verändert.');
+  const result = await list(target, Bucket);
+  requireThat(canonical(result.map(({ Key, Size }) => ({ Key, Size }))) ===
+    canonical(current.objects.map(({ Key, Size }) => ({ Key, Size }))), 'Schlüsselmenge oder Größen weichen ab.');
+  const report = { schemaVersion: 1, verifiedAt: new Date().toISOString(), count: current.count, bytes: current.bytes,
+    inventorySha256: digest(current), objectsSha256: digest(hashes), objects: hashes };
+  await atomicJson(join(directory, 'verified.json'), report);
+  return report;
+}
+
+export async function verifyManifest(target, Bucket, report) {
+  const objects = await list(target, Bucket);
+  requireThat(canonical(objects.map(o => ({ key: o.Key, size: o.Size }))) ===
+    canonical(report.objects.map(o => ({ key: o.key, size: o.size }))), 'Restore: Schlüsselmenge/Größen abweichend.');
+  for (const object of report.objects) {
+    requireThat(canonical(await readHash(target, Bucket, { Key: object.key, Size: object.size })) ===
+      canonical({ sha256: object.sha256, size: object.size, metadata: object.metadata }), 'Restore: Hash/Metadaten abweichend.');
+  }
+  return { count: report.count, bytes: report.bytes, restored: true };
+}
+
+export async function cleanupProbe(s3, Bucket, probePath) {
+  let record;
+  try { record = JSON.parse(await readFile(probePath, 'utf8')); }
+  catch (error) { if (error.code === 'ENOENT') return; throw error; }
+  requireThat(/^fedeo-contract-[0-9a-f-]{36}\//.test(record.key), 'Ungültiges Probejournal.');
+  await call(s3, 'DeleteObject', { Bucket, Key: record.key });
+  await call(s3, 'DeleteObject', { Bucket, Key: record.key + '.stream' });
+}
+
+export async function contract(s3, config, Bucket, { foreignBucket, probePath } = {}) {
+  if (probePath) await cleanupProbe(s3, Bucket, probePath);
+  const Key = `fedeo-contract-${randomUUID()}/Mandant ä + % #/Prüfung.pdf`;
+  if (probePath) await atomicJson(probePath, { key: Key });
+  const body = Buffer.from('FEDEO S3 Prüfung\n');
+  try {
+    await call(s3, 'PutObject', { Bucket, Key, Body: body, ContentType: 'application/pdf',
+      ContentDisposition: 'inline; filename="test.pdf"', CacheControl: 'private, max-age=0', Metadata: { test: 'fedeo' } });
+    const head = await call(s3, 'HeadObject', { Bucket, Key });
+    requireThat(head.ContentLength === body.length && head.Metadata.test === 'fedeo', 'HEAD-Vertrag fehlgeschlagen.');
+    const got = await call(s3, 'GetObject', { Bucket, Key });
+    requireThat(Buffer.from(await got.Body.transformToByteArray()).equals(body), 'GET-Vertrag fehlgeschlagen.');
+    // Application exports use streams. Exercise exactly the SDK environment setting
+    // applied by the pilot Compose file, including empty streams.
+    for (const bytes of [Buffer.alloc(0), Buffer.alloc(128 * 1024, 42)]) {
+      await call(s3, 'PutObject', { Bucket, Key: Key + '.stream', ContentLength: bytes.length,
+        Body: Readable.from([bytes]), ContentType: 'application/zip' });
+      const streamHead = await call(s3, 'HeadObject', { Bucket, Key: Key + '.stream' });
+      requireThat(!streamHead.ContentEncoding && streamHead.ContentLength === bytes.length, 'Streaming-Metadaten abweichend.');
+      const downloaded = await call(s3, 'GetObject', { Bucket, Key: Key + '.stream' });
+      requireThat(Buffer.from(await downloaded.Body.transformToByteArray()).equals(bytes), 'Streaming-Inhalt abweichend.');
+    }
+    const url = await getSignedUrl(s3, new S3.GetObjectCommand({ Bucket, Key }), { expiresIn: 86400 });
+    const signed = await fetch(url);
+    requireThat(signed.ok && Buffer.from(await signed.arrayBuffer()).equals(body), 'Presigned-URL fehlgeschlagen.');
+    const anonymous = await fetch(url.split('?')[0]);
+    await anonymous.arrayBuffer();
+    requireThat([401, 403].includes(anonymous.status), 'Anonymer Objektzugriff erlaubt!');
+    const wrong = client({ ...config, credentials: { accessKeyId: 'fedeo-invalid', secretAccessKey: 'fedeo-invalid-secret' } });
+    try { await denied(wrong, 'GetObject', { Bucket, Key }); } finally { wrong.destroy(); }
+    if (foreignBucket) await denied(s3, 'ListObjectsV2', { Bucket: foreignBucket });
+    await call(s3, 'DeleteObject', { Bucket, Key });
+    try { await call(s3, 'HeadObject', { Bucket, Key }); throw new Error('DELETE nicht wirksam.'); }
+    catch (error) { if (error.$metadata?.httpStatusCode !== 404) throw error; }
+  } finally { await call(s3, 'DeleteObject', { Bucket, Key }); await call(s3, 'DeleteObject', { Bucket, Key: Key + '.stream' }); }
+  return { operations: ['PUT', 'HEAD', 'GET', 'DELETE', 'SigV4', 'presigned-24h', 'anonymous-denied', 'wrong-key-denied'] };
+}
+
+async function denied(s3, operation, input) {
+  try { const value = await call(s3, operation, input); value.Body?.destroy(); }
+  catch (error) { if ([401, 403].includes(error.$metadata?.httpStatusCode)) return; throw error; }
+  throw new Error('Nicht autorisierter Zugriff wurde erlaubt.');
+}

scripts/storage/seaweedfs.json

@@ -0,0 +1,16 @@
+{
+  "image": "chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882",
+  "entrypoint": ["/bin/sh", "/etc/fedeo/start-seaweedfs.sh"],
+  "command": [
+    "server", "-dir=/data", "-master.dir=/data/master", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
+    "-master.telemetry=false", "-master.volumeSizeLimitMB=1024", "-volume.max=0", "-volume.minFreeSpace=1GiB",
+    "-filer", "-s3", "-s3.ip.bind=0.0.0.0", "-s3.port=9000", "-s3.config=/etc/fedeo/s3.json",
+    "-s3.iam=false", "-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.autoCreateBucket=false",
+    "-s3.allowDeleteBucketNotEmpty=false"
+  ],
+  "stop_grace_period": "60s",
+  "healthcheck": {
+    "test": ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:9000/ 2>&1 | grep -q ' 403 '"],
+    "interval": "5s", "timeout": "3s", "retries": 30, "start_period": "10s"
+  }
+}

scripts/storage/start-seaweedfs.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+# Never start an anonymously writable gateway when its configuration is absent,
+# truncated, or replaced. The host generated and validated this exact file.
+test -n "${FEDEO_S3_CONFIG_SHA256:-}"
+test -s /etc/fedeo/s3.json
+actual_sha256="$(sha256sum /etc/fedeo/s3.json)"
+test "${actual_sha256%% *}" = "$FEDEO_S3_CONFIG_SHA256"
+exec /usr/bin/weed "$@"

scripts/storage/sync-sdk-lock.mjs

@@ -0,0 +1,31 @@
+// Preserve the backend's exact transitive SDK graph; top-level version equality alone is insufficient.
+import { readFileSync, writeFileSync } from 'node:fs';
+const backend = JSON.parse(readFileSync(new URL('../../backend/package-lock.json', import.meta.url)));
+const manifest = JSON.parse(readFileSync(new URL('./package.json', import.meta.url)));
+const packages = { '': { name: manifest.name, version: manifest.version, dependencies: manifest.dependencies, engines: manifest.engines } };
+function resolve(from, dependency) {
+  const parts = from.split('/').filter(Boolean);
+  while (true) {
+    const candidate = [...parts, 'node_modules', dependency].join('/');
+    if (backend.packages[candidate]) return candidate;
+    if (!parts.length) return null;
+    parts.pop();
+  }
+}
+const queue = Object.keys(manifest.dependencies).map(name => resolve('', name));
+while (queue.length) {
+  const path = queue.pop();
+  if (!path) throw new Error('Missing dependency in backend lockfile');
+  if (packages[path]) continue;
+  const entry = { ...backend.packages[path] };
+  delete entry.dev;
+  packages[path] = entry;
+  for (const name of Object.keys(entry.dependencies || {})) queue.push(resolve(path, name));
+  for (const name of Object.keys(entry.optionalDependencies || {})) {
+    const resolved = resolve(path, name);
+    if (resolved) queue.push(resolved);
+  }
+}
+writeFileSync(new URL('./package-lock.json', import.meta.url), JSON.stringify({ name: manifest.name, version: manifest.version,
+  lockfileVersion: 3, requires: true, packages: Object.fromEntries(Object.entries(packages).sort(([a], [b]) => a.localeCompare(b))) }, null, 2) + '\n');
+console.log(`Copied ${Object.keys(packages).length - 1} locked backend dependencies.`);

scripts/storage/tests/Dockerfile.minio

@@ -0,0 +1,6 @@
+# Test fixture only. The pilot bundle uses the instance's already installed MinIO.
+FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32
+RUN wget -q -O /usr/local/bin/minio https://github.com/minio/minio/releases/download/RELEASE.2025-09-07T16-13-09Z/minio.linux-amd64.RELEASE.2025-09-07T16-13-09Z \
+    && echo '7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f  /usr/local/bin/minio' | sha256sum -c - \
+    && chmod 755 /usr/local/bin/minio
+ENTRYPOINT ["minio"]

scripts/storage/tests/host_integration.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Real Docker host-state test with synthetic services, never a real FEDEO instance."""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import uuid
+import sys
+
+ROOT = Path(__file__).resolve().parents[2]
+spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py")
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+def docker(*args):
+    return subprocess.check_output(["docker", *map(str, args)], text=True, stderr=subprocess.PIPE).strip()
+
+
+def fixture():
+    root = Path(tempfile.mkdtemp(prefix="fedeo-host-migration-"))
+    project = "fedeo-pilot-" + uuid.uuid4().hex[:8]
+    (root / "minio").mkdir()
+    (root / "postgres").mkdir()
+    (root / ".env").write_text("# synthetic test only\n")
+    secret = "test-with-$-interpolation"
+    env = {"S3_ENDPOINT": "http://minio:9000", "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo",
+           "S3_ACCESS_KEY": "test-user", "S3_SECRET_KEY": secret, "FEDEO_VERSION": "3.3.0",
+           "DATABASE_URL": "postgres://fedeo:test-db@db:5432/fedeo", "FEDEO_FILE_BACKEND": "s3"}
+    idle = {"image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"],
+            "restart": "unless-stopped", "networks": ["internal"],
+            "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(0)"], "interval": "1s"}}
+    import copy
+    services = {n: copy.deepcopy(idle) for n in ("backend", "frontend", "traefik", "node-exporter", "createbuckets")}
+    services["createbuckets"].update(command=["node", "-e", "process.exit(0)"], restart="no")
+    services["backend"]["environment"] = env
+    services["backend"]["image"] = "fedeo-storage-tools:pilot"
+    services["backend"]["entrypoint"] = ["node"]
+    services["backend"]["command"] = ["-e", "setInterval(()=>{},1000)"]
+    services["backend"]["depends_on"] = {"minio": {"condition": "service_healthy"}, "db": {"condition": "service_healthy"},
+                                                 "createbuckets": {"condition": "service_completed_successfully"}}
+    services["minio"] = {"image": "fedeo-minio-fixture:2025-09-07", "restart": "unless-stopped",
+        "command": ["server", "/data", "--console-address", ":9001"], "networks": ["internal"],
+        "volumes": [str(root / "minio") + ":/data"],
+        "environment": {"MINIO_ROOT_USER": "test-user", "MINIO_ROOT_PASSWORD": secret},
+        "healthcheck": {"test": ["CMD", "node", "-e", "fetch('http://127.0.0.1:9000/minio/health/live').then(r=>process.exit(r.ok?0:1))"], "interval": "1s"}}
+    services["db"] = {"image": "postgres:16-alpine", "restart": "unless-stopped", "networks": ["internal"],
+        "volumes": [str(root / "postgres") + ":/var/lib/postgresql/data"],
+        "environment": {"POSTGRES_USER": "fedeo", "POSTGRES_PASSWORD": "test-db", "POSTGRES_DB": "fedeo"},
+        "healthcheck": {"test": ["CMD", "pg_isready", "-U", "fedeo"], "interval": "1s"}}
+    config = {"name": project, "services": services, "networks": {"internal": {"name": project + "-internal"}}}
+    (root / "docker-compose.yml").write_text(json.dumps(m.literal(config)))
+    return root, project
+
+
+def seed(root, project, hidden=False):
+    code = """import { client, call } from './s3.mjs';
+const s=client({endpoint:'http://minio:9000',region:'eu-central-1',credentials:{accessKeyId:'test-user',secretAccessKey:'test-with-$-interpolation'}});
+await call(s,'CreateBucket',{Bucket:'fedeo'});
+await call(s,'PutObject',{Bucket:'fedeo',Key:'tenant/filesbyid/1/ä + % #.pdf',Body:Buffer.from('fixture'),ContentType:'application/pdf'});"""
+    if hidden:
+        code += "for(const Key of ['hidden','hidden/child']) await call(s,'PutObject',{Bucket:'fedeo',Key,Body:'fixture'});"
+    code += "s.destroy();"
+    docker("run", "--rm", "--pull", "never", "--network", project + "-internal", "--entrypoint", "node", "fedeo-storage-tools:pilot", "--input-type=module", "-e", code)
+
+
+def run(case):
+    root, project = fixture()
+    base = ["compose", "-p", project, "-f", str(root / "docker-compose.yml")]
+    try:
+        docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90")
+        seed(root, project, hidden=case == "hidden-prefix")
+        migration = m.Migration(root, "fedeo-storage-tools:pilot")
+        with migration.engine.lock():
+            if case == "hidden-prefix":
+                try:
+                    migration.plan()
+                    raise AssertionError("hidden key conflict must block planning")
+                except m.Error as error:
+                    assert "konflikt" in str(error)
+                assert not (root / ".updates/status.json").exists()
+                print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
+                return
+            migration.plan()
+            assert migration.state["count"] == 1
+            migration.migrate(True)
+            assert migration.state["phase"] == "verified"
+            assert m.u.read_json(root / ".updates/status.json")["recoveryRequired"]
+            # Simulate restart by reconstructing the process and persisted state.
+            migration = m.Migration(root)
+            if case == "rollback":
+                migration.rollback()
+                assert migration.state["phase"] == "rolled_back"
+                assert not m.u.read_json(root / ".updates/status.json").get("recoveryRequired")
+                assert migration.inspect(migration.state["containers"]["minio"]["id"])["State"]["Running"]
+            else:
+                migration.activate()
+                assert migration.state["phase"] == "complete" and migration.state["writesReleased"]
+                assert migration.state["restoreVerifiedAt"]
+                old = migration.inspect(migration.state["containers"]["minio"]["id"])
+                assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no"
+                assert migration.state["network"] not in old["NetworkSettings"]["Networks"]
+                try:
+                    migration.rollback()
+                    raise AssertionError("rollback after writes must fail")
+                except m.Error:
+                    pass
+        print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True)
+    except BaseException:
+        print("Failure evidence: " + str(root), flush=True)
+        log = root / ".updates/storage-migration/commands.log"
+        if log.exists():
+            print(log.read_text()[-8000:], flush=True)
+        ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project,
+                     "--filter", "label=com.docker.compose.service=objectstore").split()
+        for cid in ids:
+            logs = subprocess.run(["docker", "logs", "--tail", "60", cid], capture_output=True, text=True)
+            print(logs.stdout + logs.stderr, flush=True)
+        raise
+    finally:
+        ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split()
+        if ids:
+            docker("rm", "-f", *ids)
+        remaining = docker("ps", "-aq", "--filter", "label=io.fedeo.storage-job=" + str(root / ".updates/storage-migration")).split()
+        if remaining:
+            docker("rm", "-f", *remaining)
+        docker("network", "rm", project + "-internal")
+
+
+if __name__ == "__main__":
+    for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]):
+        assert case in ("hidden-prefix", "rollback", "activate")
+        run(case)

scripts/storage/tests/integration.mjs

@@ -0,0 +1,112 @@
+// Explicit integration run; never discovers or connects to an existing FEDEO stack.
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtemp, mkdir, writeFile, readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { randomUUID, createHash } from 'node:crypto';
+import { client, call, contract, inventory, transfer, verifyManifest, MAX_OBJECT_BYTES } from '../s3.mjs';
+
+const docker = (...args) => execFileSync('docker', args, { encoding: 'utf8', timeout: 120_000, stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+const spec = JSON.parse(await readFile(new URL('../seaweedfs.json', import.meta.url), 'utf8'));
+const root = await mkdtemp(join(tmpdir(), 'fedeo-storage-contract-'));
+const prefix = `fedeo-contract-${randomUUID().slice(0, 8)}`;
+const containers = [];
+const Bucket = 'fedeo';
+const credentials = { accessKeyId: 'fedeo-test-user', secretAccessKey: 'fedeo-test-secret-only' };
+const adminCredentials = { accessKeyId: 'fedeo-test-admin', secretAccessKey: 'fedeo-test-admin-secret' };
+process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED';
+const config = endpoint => ({ endpoint, region: 'eu-central-1', credentials });
+let source, target, admin;
+function run(name, ...args) { containers.push(name); return docker('run', '-d', '--pull', 'never', '--name', name, ...args); }
+function endpoint(name) { return 'http://' + docker('port', name, '9000/tcp'); }
+async function ready(s3) {
+  for (let i = 0; i < 90; i++) {
+    try { await call(s3, 'ListBuckets', {}); return; }
+    catch { await new Promise(resolve => setTimeout(resolve, 1000)); }
+  }
+  throw new Error('S3 did not become ready');
+}
+async function weed(name, data) {
+  await mkdir(join(data, 'master'), { recursive: true });
+  run(name, '-p', '127.0.0.1::9000', '-v', `${data}:/data`, '-v', `${root}/config:/etc/fedeo:ro`,
+    '-e', 'FEDEO_S3_CONFIG_SHA256=' + createHash('sha256').update(await readFile(join(root, 'config/s3.json'))).digest('hex'),
+    '--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command);
+  return endpoint(name);
+}
+try {
+  await mkdir(join(root, 'config'));
+  await writeFile(join(root, 'config/start-seaweedfs.sh'), await readFile(new URL('../start-seaweedfs.sh', import.meta.url)));
+  assert.throws(() => docker('run', '--rm', '--pull', 'never', '-v', `${root}/config:/etc/fedeo:ro`,
+    '--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command));
+  await writeFile(join(root, 'config/s3.json'), JSON.stringify({ identities: [
+    { name: 'admin', credentials: [adminCredentials], actions: ['Admin', 'Read', 'Write', 'List', 'Tagging'] },
+    { name: 'fedeo', credentials: [credentials], actions: [`Read:${Bucket}`, `Write:${Bucket}`, `List:${Bucket}`] }
+  ] }));
+  // Seaweed's credential JSON uses accessKey/secretKey rather than SDK names.
+  const auth = JSON.parse(await readFile(join(root, 'config/s3.json'), 'utf8'));
+  for (const identity of auth.identities) identity.credentials = identity.credentials.map(c => ({ accessKey: c.accessKeyId, secretKey: c.secretAccessKey }));
+  await writeFile(join(root, 'config/s3.json'), JSON.stringify(auth));
+  run(`${prefix}-source`, '-p', '127.0.0.1::9000', '-e', `MINIO_ROOT_USER=${credentials.accessKeyId}`,
+    '-e', `MINIO_ROOT_PASSWORD=${credentials.secretAccessKey}`, process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07',
+    'server', '/data', '--console-address', ':9001');
+  const sourceConfig = config(endpoint(`${prefix}-source`));
+  source = client(sourceConfig);
+  await ready(source);
+  await call(source, 'CreateBucket', { Bucket });
+  await contract(source, sourceConfig, Bucket);
+  const targetConfig = config(await weed(`${prefix}-target`, join(root, 'target')));
+  target = client(targetConfig);
+  admin = client({ ...targetConfig, credentials: adminCredentials });
+  await ready(admin);
+  await call(admin, 'CreateBucket', { Bucket });
+  await call(admin, 'CreateBucket', { Bucket: 'foreign-private' });
+  await contract(target, targetConfig, Bucket, { foreignBucket: 'foreign-private' });
+  // Confirm unauthenticated management ports are not reachable through the container IP.
+  const ip = JSON.parse(docker('inspect', `${prefix}-target`))[0].NetworkSettings.Networks.bridge.IPAddress;
+  for (const port of [9333, 19333, 8888, 18888, 8080, 18080, 8181, 9101]) {
+    assert.throws(() => docker('exec', `${prefix}-source`, 'node', '-e',
+      `const s=require('node:net').connect(${port},${JSON.stringify(ip)});s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));s.setTimeout(1500,()=>process.exit(1))`));
+  }
+  for (let i = 0; i < 1003; i++) await call(source, 'PutObject', { Bucket, Key: `tenant/filesbyid/${i}/ä + % #.txt`,
+    Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } });
+  await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' });
+  await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES,
+    Body: Readable.from((function* () { for (let i = 0; i < 256; i++) yield Buffer.alloc(1024 * 1024, i); })()),
+    ContentType: 'application/zip', ContentDisposition: 'attachment; filename="large.zip"',
+    CacheControl: 'private', ContentLanguage: 'de', Expires: new Date('2030-01-01T00:00:00Z') });
+  const before = await inventory(source, Bucket);
+  assert.equal(before.count, 1005);
+  await call(source, 'PutObject', { Bucket, Key: 'unsupported-marker/', Body: '' });
+  await assert.rejects(inventory(source, Bucket), /Schlüssel/);
+  await call(source, 'DeleteObject', { Bucket, Key: 'unsupported-marker/' });
+  const start = Date.now();
+  const report = await transfer(source, target, Bucket, join(root, 'work'));
+  await call(target, 'PutObject', { Bucket, Key: 'empty', Body: 'corrupt' });
+  await assert.rejects(transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true }), /Integritätsfehler/);
+  await transfer(source, target, Bucket, join(root, 'work')); // Resume repairs and re-reads, never trusts journal alone.
+  await transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true });
+  docker('stop', '-t', '60', `${prefix}-target`);
+  docker('run', '--rm', '--pull', 'never', '-v', `${root}:/test`, '--entrypoint', '/bin/sh', spec.image,
+    '-ec', 'tar -cpf /test/backup.tar -C /test/target .; mkdir /test/restored; tar -xpf /test/backup.tar -C /test/restored');
+  const restoredConfig = config(await weed(`${prefix}-restored`, join(root, 'restored')));
+  const restored = client(restoredConfig);
+  const restoredAdmin = client({ ...restoredConfig, credentials: adminCredentials });
+  try {
+    await ready(restoredAdmin);
+    await verifyManifest(restored, Bucket, report);
+    await contract(restored, restoredConfig, Bucket, { foreignBucket: 'foreign-private' });
+  } finally { restored.destroy(); restoredAdmin.destroy(); }
+  console.log(JSON.stringify({ result: 'passed', objects: report.count, bytes: report.bytes,
+    seconds: (Date.now() - start) / 1000, seaweed: spec.image, source: docker('image', 'inspect',
+      process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', '--format', '{{.Id}}'),
+    evidence: root }, null, 2));
+} catch (error) {
+  for (const name of containers) { try { console.error(docker('logs', '--tail', '12', name)); } catch {} }
+  throw error;
+} finally {
+  source?.destroy(); target?.destroy(); admin?.destroy();
+  for (const name of containers.reverse()) { try { docker('rm', '-f', name); } catch {} }
+  // Preserve the isolated backup and report for inspection; no production paths are touched.
+}

scripts/storage/tests/keys.test.mjs

@@ -0,0 +1,37 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { validateKeys, MAX_OBJECT_BYTES, metadata } from '../s3.mjs';
+import { cleanupProbe, atomicJson } from '../s3.mjs';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+test('accepts FEDEO keys, special characters, empty objects and size boundary', () => {
+  validateKeys([{ Key: 'tenant/filesbyid/123/ä + %#.pdf', Size: 0 }, { Key: 'export.zip', Size: MAX_OBJECT_BYTES }]);
+});
+test('blocks lossy paths and file/directory collisions regardless of ordering', () => {
+  for (const Key of ['/a', 'a/', 'a//b', 'a/../b', './a', 'a\\b', 'a\nb'])
+    assert.throws(() => validateKeys([{ Key, Size: 1 }]));
+  assert.throws(() => validateKeys([{ Key: 'a/b', Size: 1 }, { Key: 'a', Size: 1 }]));
+  assert.throws(() => validateKeys([{ Key: 'a', Size: MAX_OBJECT_BYTES + 1 }]));
+});
+test('rejects encryption and nonportable attributes rather than dropping them', () => {
+  for (const value of [{ ServerSideEncryption: 'AES256' }, { ObjectLockMode: 'GOVERNANCE' },
+    { StorageClass: 'GLACIER' }, { WebsiteRedirectLocation: '/other' }]) assert.throws(() => metadata(value));
+  assert.deepEqual(metadata({ ContentType: 'application/pdf', Metadata: { custom: 'value' } }),
+    { ContentType: 'application/pdf', Metadata: { custom: 'value' } });
+});
+
+test('interrupted contract cleanup deletes only its two journalled probe objects', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'fedeo-probe-test-'));
+  try {
+    const path = join(directory, 'probe.json');
+    const key = 'fedeo-contract-11111111-1111-1111-1111-111111111111/test';
+    await atomicJson(path, { key });
+    const inputs = [];
+    await cleanupProbe({ send: async command => { inputs.push(command.input); return {}; } }, 'bucket', path);
+    assert.deepEqual(inputs, [{ Bucket: 'bucket', Key: key }, { Bucket: 'bucket', Key: key + '.stream' }]);
+    await atomicJson(path, { key: 'real-business-file.pdf' });
+    await assert.rejects(cleanupProbe({ send: async () => { throw new Error('must not send'); } }, 'bucket', path), /Probejournal/);
+  } finally { await rm(directory, { recursive: true, force: true }); }
+});

scripts/storage/tests/test_host.py

@@ -0,0 +1,125 @@
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import Mock
+
+spec = importlib.util.spec_from_file_location("storage_migration", Path(__file__).resolve().parents[2] / "selfhost-storage-migrate.py")
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+class HostSafetyTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        for name in ("minio", "postgres"):
+            (self.root / name).mkdir()
+        self.config = {"services": {n: {} for n in ("db", "minio", "createbuckets", "backend", "frontend", "traefik", "node-exporter")}}
+        for name, path, target in (("minio", "minio", "/data"), ("db", "postgres", "/var/lib/postgresql/data")):
+            self.config["services"][name]["volumes"] = [{"type": "bind", "source": str(self.root / path), "target": target}]
+        self.config["services"]["minio"].update(networks={"internal": {}}, environment={"MINIO_ROOT_USER": "user", "MINIO_ROOT_PASSWORD": "secret"})
+        self.config["services"]["db"]["environment"] = {"POSTGRES_USER": "u", "POSTGRES_PASSWORD": "p", "POSTGRES_DB": "db"}
+        self.config["services"]["backend"]["environment"] = {"DATABASE_URL": "postgres://u:p@db:5432/db", "S3_ENDPOINT": "http://minio:9000",
+            "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo", "S3_ACCESS_KEY": "user", "S3_SECRET_KEY": "secret"}
+
+    def test_rejects_external_or_unbacked_storage_and_secret_drift(self):
+        m.validate_config(self.config, self.root)
+        mutations = [lambda c: c["services"]["minio"].update(ports=["9000:9000"]),
+            lambda c: c["services"]["backend"]["environment"].update(S3_ENDPOINT="https://external.example"),
+            lambda c: c["services"]["backend"]["environment"].update(S3_SECRET_KEY="different"),
+            lambda c: c["services"]["minio"]["volumes"][0].update(source="/elsewhere"),
+            lambda c: c["services"].update(unknown={}),
+            lambda c: c["services"]["backend"]["environment"].update(DATABASE_URL="postgres://u:p@db:5432/other")]
+        for mutate in mutations:
+            config = copy.deepcopy(self.config)
+            mutate(config)
+            with self.assertRaises(m.Error):
+                m.validate_config(config, self.root)
+
+    def test_rejects_symlink_source(self):
+        (self.root / "minio").rmdir()
+        (self.root / "minio").symlink_to(self.root / "postgres")
+        with self.assertRaises(m.Error):
+            m.validate_config(self.config, self.root)
+
+    def test_cold_index_detects_objects_hidden_by_minio_listing(self):
+        source = self.root / "minio"
+        (source / ".minio.sys").mkdir()
+        (source / ".minio.sys/format.json").write_text(json.dumps({"format": "xl-single", "xl": {"version": "3"}}))
+        (source / "fedeo/a").mkdir(parents=True)
+        (source / "fedeo/a/xl.meta").write_bytes(b"opaque")
+        m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}]}, exact=True)
+        (source / "fedeo/a/b").mkdir()
+        (source / "fedeo/a/b/xl.meta").write_bytes(b"hidden-from-S3-LIST")
+        with self.assertRaisesRegex(m.Error, "Konflikt|konflikt"):
+            m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}]}, exact=True)
+        (source / "fedeo/a/b/xl.meta").unlink()
+        with self.assertRaisesRegex(m.Error, "weichen ab"):
+            m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}, {"Key": "missing"}]}, exact=True)
+
+    def test_update_and_migration_share_exclusive_lock(self):
+        a, b = m.Migration(self.root), m.Migration(self.root)
+        with a.engine.lock():
+            with self.assertRaises(m.Error):
+                with b.engine.lock():
+                    self.fail("lock must be exclusive")
+
+    def test_failed_freeze_keeps_persistent_recovery_guard(self):
+        migration = m.Migration(self.root)
+        migration.state = {"startedAt": "test", "writesReleased": False, "containers": {"backend": {"id": "backend"}}}
+        def fail(*args, **kwargs):
+            state = m.u.read_json(migration.engine.home / "status.json")
+            self.assertTrue(state["recoveryRequired"])
+            self.assertEqual(m.u.read_json(migration.home / "state.json")["phase"], "freezing")
+            raise m.Error("power loss")
+        migration.docker = fail
+        with self.assertRaises(m.Error):
+            migration.freeze()
+        self.assertFalse(m.u.read_json(migration.home / "state.json")["writesReleased"])
+
+    def test_backup_corruption_prevents_progress(self):
+        migration = m.Migration(self.root)
+        backup = migration.home / "backup"
+        backup.mkdir()
+        (backup / "files.tar").write_bytes(b"valid")
+        m.u.write_json(backup / "complete.json", {"sha256": {"files.tar": m.sha_file(backup / "files.tar")}})
+        migration.check_backup()
+        (backup / "files.tar").write_bytes(b"corrupt")
+        with self.assertRaises(m.Error):
+            migration.check_backup()
+
+    def test_start_failure_is_already_past_rollback_boundary(self):
+        migration = m.Migration(self.root)
+        migration.state = {"startedAt": "test", "phase": "verified", "network": "internal", "writesReleased": False,
+                           "containers": {"minio": {"id": "old"}}}
+        for method in ("assert_frozen", "verify_original", "check_backup", "cleanup_helper", "helper", "docker", "compose"):
+            setattr(migration, method, Mock())
+        migration.inspect = lambda _: {"NetworkSettings": {"Networks": {}}}
+        migration.restore_test = Mock()
+        migration.runner = Mock()
+        (migration.home / "target.json").write_text('{}')
+        def fail_start():
+            self.assertTrue(m.u.read_json(migration.home / "state.json")["writesReleased"])
+            raise m.Error("backend did not start")
+        migration.start_application = fail_start
+        from unittest.mock import patch
+        (migration.home / "credentials.json").write_text('{"bucket":"fedeo"}')
+        with patch.object(m, "validate_minio_listing"):
+            (migration.home / "copy").mkdir()
+            (migration.home / "copy/inventory.json").write_text('{}')
+            with self.assertRaises(m.Error):
+                migration.activate()
+        with self.assertRaises(m.Error):
+            migration.rollback()
+
+    def test_compose_roundtrip_preserves_dollar_signs(self):
+        value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]}
+        self.assertEqual(m.resolved(m.literal(value)), value)
+
+
+if __name__ == "__main__":
+    unittest.main()