From 46ee819029bf233c498a169572051c9e042de0aa Mon Sep 17 00:00:00 2001 From: flfeders Date: Sun, 27 Sep 2026 17:22:13 +0200 Subject: [PATCH] feat(storage): add verified offline MinIO migration pilot --- docs/README.md | 1 + docs/minio-austausch-konzept.md | 5 + docs/minio-umstellung-pilot.md | 226 +++ package.json | 1 + scripts/selfhost-storage-migrate.py | 607 +++++++ scripts/storage/.dockerignore | 2 + scripts/storage/Dockerfile | 6 + scripts/storage/README.md | 44 + scripts/storage/build-pilot.sh | 39 + scripts/storage/cli.mjs | 39 + scripts/storage/package-lock.json | 1740 +++++++++++++++++++++ scripts/storage/package.json | 12 + scripts/storage/pilot.py | 51 + scripts/storage/s3.mjs | 283 ++++ scripts/storage/seaweedfs.json | 16 + scripts/storage/start-seaweedfs.sh | 9 + scripts/storage/sync-sdk-lock.mjs | 31 + scripts/storage/tests/Dockerfile.minio | 6 + scripts/storage/tests/host_integration.py | 135 ++ scripts/storage/tests/integration.mjs | 112 ++ scripts/storage/tests/keys.test.mjs | 37 + scripts/storage/tests/test_host.py | 125 ++ 22 files changed, 3527 insertions(+) create mode 100644 docs/minio-umstellung-pilot.md create mode 100644 scripts/selfhost-storage-migrate.py create mode 100644 scripts/storage/.dockerignore create mode 100644 scripts/storage/Dockerfile create mode 100644 scripts/storage/README.md create mode 100644 scripts/storage/build-pilot.sh create mode 100644 scripts/storage/cli.mjs create mode 100644 scripts/storage/package-lock.json create mode 100644 scripts/storage/package.json create mode 100644 scripts/storage/pilot.py create mode 100644 scripts/storage/s3.mjs create mode 100644 scripts/storage/seaweedfs.json create mode 100644 scripts/storage/start-seaweedfs.sh create mode 100644 scripts/storage/sync-sdk-lock.mjs create mode 100644 scripts/storage/tests/Dockerfile.minio create mode 100644 scripts/storage/tests/host_integration.py create mode 100644 scripts/storage/tests/integration.mjs create mode 100644 scripts/storage/tests/keys.test.mjs create mode 100644 scripts/storage/tests/test_host.py diff --git a/docs/README.md b/docs/README.md index 9405a470..7b410691 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,4 +8,5 @@ Diese Dokumentation unterstützt dich bei der täglichen Nutzung von FEDEO. - [Fach- und Technikkonzept für das Lagersystem](./lagersystem-konzept.md) - [Zentraler Push-Server für Selfhost-Instanzen](./zentraler-push-server.md) - [Konzept zum Austausch von MinIO](./minio-austausch-konzept.md) +- [MinIO-Umstellung: ausführbarer Pilot](./minio-umstellung-pilot.md) - [Konzept für föderierte Chats und externe Projektteilnehmer](./chat-foederation-konzept.md) diff --git a/docs/minio-austausch-konzept.md b/docs/minio-austausch-konzept.md index c9a27888..e126acce 100644 --- a/docs/minio-austausch-konzept.md +++ b/docs/minio-austausch-konzept.md @@ -2,6 +2,11 @@ Stand: 25. September 2026. Status: Entscheidungsvorschlag auf Basis des FEDEO-Codes bei Commit `29155f91` und der unten verlinkten Herstellerquellen. Eine Migration, ein Kompatibilitätstest mit einem Ersatzserver oder eine Änderung an einer laufenden Instanz wurde für dieses Konzept nicht durchgeführt. +**Fortschreibung vom 27. September 2026:** Ein ausführbarer, isoliert getesteter +Migrationspilot ist hinzugekommen. Durchführung, Testnachweise und noch offene +Freigaben stehen in [MinIO-Umstellung: Pilot](minio-umstellung-pilot.md). +Die folgenden Abschnitte bleiben das Zielbild für den vollständigen Standardbetrieb. + ## 1. Empfehlung und Umfang **SeaweedFS Open Source wird der bevorzugte Kandidat für den FEDEO-Dateispeicher.** Ziel ist ein einzelner Container auf dem bisherigen Docker-Host mit S3-Schnittstelle, privaten Buckets und persistentem lokalen Speicher. Die Freigabe erfolgt erst nach den unter Abschnitt 10 beschriebenen Tests mit einem festgelegten Release und Image-Digest. diff --git a/docs/minio-umstellung-pilot.md b/docs/minio-umstellung-pilot.md new file mode 100644 index 00000000..834a6642 --- /dev/null +++ b/docs/minio-umstellung-pilot.md @@ -0,0 +1,226 @@ +# MinIO → SeaweedFS: betreuter Pilot auf einer Installer-Instanz + +Stand: 27. September 2026. Dieser Ablauf ergänzt das +[Austauschkonzept](minio-austausch-konzept.md) um ausführbare Werkzeuge. +Er übernimmt den Speicher einer vorhandenen Installation, beispielsweise 3.3.0, +mit unveränderten Anwendungsimages und ohne Datenbankschemamigration. + +**Vor dem ersten Wartungsfenster nur `load-images` und `plan` ausführen.** +Die Vorprüfung ermittelt die bislang unbekannten Größen und liefert eine +kompakte Ausgabe zum gemeinsamen Prüfen. Sie stoppt keine Anwendungscontainer. +Ein Zugriff auf die konkrete Kundeninstanz wurde bei der Entwicklung nicht ausgeführt. + +## Was der Pilot leistet und wo seine Grenze liegt + +- Vorhandenes MinIO verwenden; kein Abruf von `minio/minio` oder `mc` erforderlich. +- Feste SeaweedFS-Version 4.47; Offline-Images, SHA-256-geprüftes Paket. +- Alle Objekte des einzigen FEDEO-Buckets übernehmen, inklusive Exporte und + Objekte ohne aktuellen Datenbankverweis. Schlüssel und Metadaten bleiben erhalten. +- Alle bekannten Schreiber anhalten, ihre automatischen Neustarts sperren, + PostgreSQL dumpen und MinIO kalt sichern. Die alten Daten bleiben erhalten. +- Kopie, vollständigen Inhaltsvergleich und tatsächlichen SeaweedFS-Restore in + ein frisches Verzeichnis prüfen. Unbekannte Zielobjekte werden nicht gelöscht. +- Den S3-Vertrag zusätzlich mit dem **tatsächlich installierten Backendimage** + ausführen. Dabei startet nur das Prüfprogramm, kein FEDEO-Backend. +- Erst nach diesen Prüfungen den Alias `minio` exklusiv an SeaweedFS vergeben + und die unveränderte Anwendung wieder starten. + +**Der Pilot ist noch kein neuer Standardinstaller und kein reguläres +FEDEO-Release.** Der bisherige Anwendungs-Updater kennt nur MinIO. Seine Aufträge +bleiben nach der Umstellung bewusst über `recoveryRequired` gesperrt, bis ein +getesteter Provider-Adapter installiert wird. Diesen Zustand nicht mit dem alten +`resolve` oder durch Löschen der Statusdatei aufheben. Ein alter Installer oder +ein altes Release-Compose darf die umgestellte Installation nicht überschreiben. + +FEDEO 3.3.0 kann im Systemstatus weiterhin den MinIO-spezifischen Health-Endpunkt +abfragen und dadurch den Dateispeicher als gestört anzeigen. Die tatsächliche +Funktionsprüfung erfolgt im Pilot über signierte S3-Operationen. Herstellerneutrale +Oberflächenchecks, eine Migrationskarte in der Oberfläche, automatische Updates +auf SeaweedFS und der Standardinstaller sind nachfolgende Arbeitspakete. +Wer diese Einschränkungen noch nicht im Betrieb tragen möchte, führt zunächst +nur die Vorprüfung durch und bleibt mit der Instanz auf MinIO. + +## Voraussetzungen, die der Plan prüft + +Linux amd64, Python ab 3.10, lokaler Docker-Daemon, Docker Compose und `tar`/`du`. +Die Installation muss laufen und aus dem Installer stammen: lokale Verzeichnisse +`postgres/` und `minio/`, interner Endpunkt `http://minio:9000`, S3-Zugang wie beim +Installer aus `MINIO_ROOT_*`, keine externen Secret-Provider, keine zusätzlichen +MinIO-Ports, Datenmounts oder unbekannten Container im internen Netz. +Bekannte lokale Matrix-Dienste werden erhalten, während der Kopie angehalten und +mit ihren Daten kalt gesichert. Andere Zusatzdienste/Compose-Overrides blockieren +den Pilot und benötigen einen angepassten Plan. + +Der S3-Prüfer blockiert zusätzliche Buckets, Versionierung einschließlich alter +Versionen/Delete-Marker, Object Lock, konfigurierte Serververschlüsselung, +Objekt-/Bucket-Tags, eigene Policies/ACLs, Lifecycle, Replikation, Benachrichtigungen, +CORS und offene Multipart-Uploads. Unklare S3-Fehler werden nicht als „nicht vorhanden“ +gewertet. Datei-/Verzeichniskonflikte und normalisierungsgefährdete Schlüssel wie +`a//b`, `a/../b` und Verzeichnismarker blockieren ebenfalls. + +Die bekannten lokalen MinIO-Metadaten werden zusätzlich geprüft (`xl-single`, +Metadatenformat 3). MinIO kann bei gleichzeitigem `a` und `a/b` einen Schlüssel +aus der S3-Liste verbergen, obwohl er per GET erreichbar bleibt. Bei gestopptem +MinIO muss deshalb die Menge lokaler `xl.meta`-Objektpfade exakt mit dem S3-Inventar +übereinstimmen. Andere Speicherformate oder Abweichungen blockieren die Umschaltung. + +Getestete Pilotgrenzen: **200.000 Objekte, höchstens 256 MiB je Objekt**. +Die Kapazitätsabschätzung berücksichtigt Quellsicherung, neues und wiederhergestelltes +Ziel, Metadatenreserve pro Objekt, DB-Dump, Offline-Images, Spool und 5 GiB Reserve. +Nach dem Schreibstopp wird das Inventar erneut ermittelt. Es gibt keine Annahme, +dass eine unbekannte freie Kapazität ausreichend ist. + +Externe Schreiber lassen sich nicht allein anhand von Docker erkennen. Vor +`migrate --writers-exclusive` muss feststehen, dass keine anderen Anwendungen, +direkten DB-/S3-Clients oder manuellen Compose-Aufträge parallel schreiben. + +## Paket erstellen und auf den Host übertragen + +Im Entwicklungscheckout mit Docker: + +```sh +bash scripts/storage/build-pilot.sh +``` + +Ergebnis: `.release/storage-pilot.tar.gz` samt `.sha256` und entpacktem Paket. +Das Archiv enthält Hostprogramme, Dokumentation und beide benötigten Zielimages. +Es enthält keine Instanz-Zugangsdaten, keine Kundendateien und kein neues MinIO. +Die Build-Phase darf Images herunterladen; alle Schritte auf dem Zielhost starten +Container mit `--pull never`. + +Archiv und Prüfsumme über den üblichen Administrationsweg auf den Docker-Host +kopieren. Dort die tatsächliche Prüfsumme mit der mitgelieferten vergleichen, +beispielsweise bei Ablage unter `/root`: + +```sh +cd /root +sha256sum storage-pilot.tar.gz +tar -xzf storage-pilot.tar.gz +cd storage-pilot +sudo python3 pilot.py load-images +sudo python3 pilot.py plan --directory /opt/fedeo +``` + +Die Ausgabe von `plan` kann in den Supportdialog kopiert werden. Sie enthält +Phase, Anwendungsversion, Objektanzahl, Objektbytes und benötigten/verfügbaren +Platz; keine Passwörter oder Dateinamen. Falls der Installer in ein anderes +Verzeichnis installiert wurde, bei jedem Befehl denselben Pfad angeben. + +Ein Plan verändert keine Quelldateien oder fachlichen Daten. Er legt geschützte +Migrationsdateien unter `.updates/storage-migration/` und einen lokalen Image-Tag +zur Aufbewahrung des Hilfsimages an. Die `storage/`-Datenablage entsteht erst beim +Migrationslauf. Das Paket bleibt bis zum Abschluss verfügbar; nicht mitten im +Auftrag neu bauen oder austauschen. + +## Wartungsfenster: kopieren, prüfen, dann freigeben + +Erst nach Prüfung der Planausgabe und Abstimmung des Wartungsfensters: + +```sh +sudo python3 pilot.py migrate --directory /opt/fedeo --writers-exclusive +sudo python3 pilot.py status --directory /opt/fedeo +``` + +`migrate` endet bei `phase: verified`. Die Anwendung ist dann **weiter angehalten**. +Es hat noch keine fachlichen Schreibzugriffe auf SeaweedFS gegeben. +Der Lauf besitzt dieselbe exklusive Sperre wie der Host-Updater. + +Die Umschaltung ist ein eigener Befehl: + +```sh +sudo python3 pilot.py activate --directory /opt/fedeo +``` + +`activate` liest Quelle und Ziel nochmals vollständig, stoppt MinIO, entfernt +dessen internen Netzwerkanschluss, sichert SeaweedFS kalt und startet einen +separaten Restore-Container mit frisch entpackten Daten. Alle Objekt-Hashes und +Metadaten müssen dort stimmen. Danach erhält das aktive Ziel allein den Alias +`minio`, und die S3-Vertragstests laufen erneut, auch mit dem Original-Backend-SDK. + +**Vor dem Start irgendeines fachlichen Schreibers** persistiert das Programm +`writesReleased: true`. Erst danach werden Backend/Hintergrunddienste und zuletzt +der öffentliche Proxy gestartet. Ein Fehler beim Neustart führt nicht zu einem +automatischen Rückfall auf den nun möglicherweise veralteten Quellspeicher. + +Anschließend auf der Instanz prüfen: Anmeldung, bestehender Download, PDF-Vorschau, +neuer Upload mit erneutem Download, WebDAV, DATEV/SEPA und Mandantenexport/-import. +Bei zwei Testmandanten auch die unveränderten Zugriffsgrenzen kontrollieren. +Bereits ausgegebene S3-Links sind zusätzlich über ihren tatsächlichen Zugriffsweg +zu prüfen; ein interner Docker-Alias macht interne URLs nicht öffentlich erreichbar. + +## Fehler und Wiederaufnahme + +| Zustand | Vorgehen | +| --- | --- | +| `planned` | Instanz läuft noch; Wartungsfenster vorbereiten. | +| `freezing`, `backing_up`, `backed_up`, `copying`, `verifying`, `verified` | `migrate --writers-exclusive` setzt fort. Quelle, Schreibstopp, Backups und Inhalte werden erneut geprüft. | +| `cutover`, `restore_testing` | `activate` setzt die noch nicht freigegebene Umschaltung fort; alternativ Rückkehr. | +| `released` nach fehlgeschlagenem Start | `resume-start` versucht denselben Zielstand erneut. Keine einfache Rückkehr mehr. | +| `complete` | SeaweedFS aktiv; alte Daten und Backups weiter aufbewahren. | +| `rolled_back` | MinIO wieder aktiv. Vor einem neuen Versuch einen neuen betreuten Plan anlegen; bestehende Beweise nicht löschen. | + +Vor der Schreibfreigabe ist die Rückkehr ausführbar: + +```sh +sudo python3 pilot.py rollback --directory /opt/fedeo +``` + +Sie stoppt das Ziel, stellt Compose, den alten exklusiven Netzwerkanschluss und +die ursprünglichen Neustartrichtlinien wieder her und startet die vorher laufenden +Container. Quelldaten und DB bleiben unverändert. Nach `writesReleased: true` +verweigert das Programm diesen Befehl. Dann braucht eine Rückkehr eine gesonderte +Rückmigration einschließlich Löschungen oder die bewusste Wiederherstellung eines +zusammengehörigen Backups. + +## Protokolle, Sicherungen und Betriebsdaten + +`status` ist die kompakte Ausgabe ohne Geheimnisse. Geschützte Einzelheiten liegen +unter `/opt/fedeo/.updates/storage-migration/`: + +- `state.json`: Phase, Container-/Image-Identitäten und dauerhafte Schreibgrenze. +- `commands.log`, `progress.jsonl`: Hostfehler und letzter Werkzeugfortschritt. +- `copy/inventory.json`, `copy/objects.jsonl`, `copy/verified.json`: vollständiges + Inventar und überprüfte Objekt-Hashes. Diese Dateien enthalten Dateinamen/Metadaten. +- `backup/database.dump`, `globals.sql`, `files.tar`, `images.tar`, `complete.json`: + zusammengehörige Quellsicherung mit Prüfsummen. Der DB-Dump wird mit `pg_restore --list` + geprüft; ein vollständiger fachlicher DB-Restore bleibt eine gesonderte DR-Prüfung. +- `backup/seaweedfs.tar`: kalt gesicherter neuer Speicher samt Konfiguration; + sein SHA-256 und erfolgreicher echter Restore werden im Zustand festgehalten. + +Die neue persistente Ablage ist `storage/seaweedfs/`, die geschützte Konfiguration +`storage/config/`. SeaweedFS bindet Verwaltung, Filer und Volume-APIs an Loopback +im Container; im internen Docker-Netz liegt ausschließlich S3 auf Port 9000. +Ein Startwrapper verweigert fehlende oder veränderte Zugangskonfiguration. +Die Anwendungsidentität erhält nur Lese-, Schreib- und Listenrechte für ihren Bucket. + +Für spätere Sicherungen **SeaweedFS anhalten und `storage/` vollständig sichern**, +einschließlich Filer-Metadaten, Masterzustand, Objektdaten, Konfiguration und passendem +Image. Die DB-Sicherung muss zum selben Schreibstopp gehören. Die Sicherungen auf +demselben Host anschließend auf das vorgesehene externe Backupziel übernehmen. +MinIO-Daten, alte Container, Test-Restores und Offline-Images werden vom Pilot +nicht automatisch gelöscht. + +## Nachgewiesene Tests und ausstehende Freigaben + +Der lokale S3-Test verwendet das offizielle MinIO-Release-Binary vom 7. September +2025 und SeaweedFS 4.47 mit festem Digest. 1.005 Objekte / 268.446.382 Bytes, +einschließlich 256-MiB-Grenzdatei, Pagination, Sonderzeichen, Metadaten, leeren +Dateien, Streaming, 24-Stunden-Signaturen, fremdem Bucket und ungültigen Zugängen +werden geprüft. Beschädigte Zielobjekte werden erkannt und beim Fortsetzen +erneut kopiert; der kalte Restore wird vollständig gelesen und verglichen. +Ein Lauf der Kopier-/Prüf-/Restore-Sequenz dauerte lokal ungefähr 65 Sekunden; +Testhost: Linux, 16 logische CPUs, rund 29 GiB RAM für Docker, lokaler NVMe-Speicher. +Dies ist keine Laufzeitzusage für die Instanz. + +Das Hilfs-Lockfile enthält den übernommenen vollständigen SDK-Abhängigkeitsbaum +des Backends, nicht nur dieselbe oberste Versionsnummer. Beim Pilot wird außerdem +das konkrete installierte Backendimage gegen das Ziel geprüft. + +Die Host-Integrationstests verwenden echte Speicher- und PostgreSQL-Container, +aber künstliche Anwendungsprozesse. Rückkehr vor Schreibfreigabe sowie vollständige +Umschaltung mit Restore und gesperrter Rückkehr nach Schreibfreigabe wurden erfolgreich +durchlaufen. Die fachliche Abnahme von FEDEO auf der +konkreten Instanz, ein physischer Hostneustart während jeder Phase, das reguläre +Anwendungsupdate auf SeaweedFS und die vollständige Wiederherstellung auf einem +anderen Host sind damit noch nicht nachgewiesen. Deshalb bleibt dieser Stand ein +betreuter Pilot; die allgemeinen Freigabekriterien des Konzepts gelten weiter. diff --git a/package.json b/package.json index 1c9dddf7..8d5f9d88 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "test": "node --test scripts/release/tests/*.test.mjs && npm run test:update", "test:update": "python3 -B -m unittest discover -s scripts/update/tests -v", + "test:storage": "npm test --prefix scripts/storage && python3 -B -m unittest discover -s scripts/storage/tests -p 'test_*.py' -v", "commits:check": "node scripts/release/cli.mjs check", "release:preview": "node scripts/release/cli.mjs preview", "release:ci": "node scripts/release/ci.mjs" diff --git a/scripts/selfhost-storage-migrate.py b/scripts/selfhost-storage-migrate.py new file mode 100644 index 00000000..14b4a80b --- /dev/null +++ b/scripts/selfhost-storage-migrate.py @@ -0,0 +1,607 @@ +#!/usr/bin/env python3 +"""Explicit, offline-capable pilot migration of an installer stack; Python 3.10+.""" +import argparse +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import secrets +import shutil +import sys +import time +import urllib.parse + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("fedeo_update", HERE / "selfhost-update.py") +u = importlib.util.module_from_spec(spec) +spec.loader.exec_module(u) +Error = u.UpdateError +GIB = 1024 ** 3 + + +class TransientStorageError(Error): + pass + + +def require(condition, message): + if not condition: + raise Error(message) + + +def sha_file(path): + h = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def literal(value): + """Resolved Compose JSON still undergoes interpolation when used as a file.""" + if isinstance(value, str): + return value.replace("$", "$$") + if isinstance(value, list): + return [literal(v) for v in value] + if isinstance(value, dict): + return {k: literal(v) for k, v in value.items()} + return value + + +def resolved(value): + """Compose config serializes literal dollar signs escaped for another Compose read.""" + if isinstance(value, str): + return value.replace("$$", "$") + if isinstance(value, list): + return [resolved(v) for v in value] + if isinstance(value, dict): + return {k: resolved(v) for k, v in value.items()} + return value + + +def validate_minio_listing(root, bucket, inventory, *, exact=False): + """MinIO can hide a/b from LIST when a exists, while GET a/b still works. + + Only inspect paths in the known single-drive layout; never decode or modify + xl.meta. A cold equality check guards the completeness of the S3 inventory. + """ + source = root / "minio" + format_info = u.read_json(source / ".minio.sys/format.json", {}) + require(format_info.get("format") == "xl-single" and format_info.get("xl", {}).get("version") == "3", + "MinIO-Datenformat ist für den zusätzlichen Vollständigkeitsabgleich nicht freigegeben.") + bucket_root = source / bucket + physical = {str(p.parent.relative_to(bucket_root)) for p in bucket_root.rglob("xl.meta") if p.is_file()} + for key in physical: + parts = key.split("/") + require(all("/".join(parts[:n]) not in physical for n in range(1, len(parts))), + "MinIO enthält einen Datei-/Verzeichniskonflikt; die S3-Auflistung kann Objekte verbergen.") + if exact: + require(physical == {o["Key"] for o in inventory["objects"]}, + "Kalte MinIO-Metadaten und S3-Inventar weichen ab; keine vollständige Übernahme nachweisbar.") + + +def validate_config(config, root): + services = config.get("services", {}) + required = {"db", "minio", "createbuckets", "backend", "frontend", "traefik", "node-exporter"} + require(required <= services.keys() and not (services.keys() - required - u.LEGACY_MATRIX_SERVICES), + "Nur der Installer-Stack wird unterstützt; zusätzliche/fehlende Dienste zuerst klären.") + for service in services.values(): + require(not any(service.get(k) for k in ("build", "profiles", "secrets", "configs", "volumes_from")), + "Builds, Profile und externe Konfigurationen benötigen einen eigenen Migrationsplan.") + store = services["minio"] + require(not store.get("ports") and set(store.get("networks", {})) == {"internal"}, + "MinIO muss ausschließlich im internen Docker-Netz erreichbar sein.") + require(not services["db"].get("ports"), "Veröffentlichte Datenbankports benötigen einen gesonderten Schreiber-Nachweis.") + require(set(store.get("environment", {})) == {"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"}, + "Zusätzliche MinIO-Konfiguration benötigt eine gesonderte Prüfung.") + for name, target, directory in (("minio", "/data", "minio"), ("db", "/var/lib/postgresql/data", "postgres")): + mounts = services[name].get("volumes", []) + require(len(mounts) == 1 and mounts[0].get("type") == "bind" and mounts[0].get("target") == target + and mounts[0].get("source") == str(root / directory), "Standard-Datenpfade fehlen: " + name) + require((root / directory).is_dir() and (root / directory).resolve() == root / directory, + "Datenpfad fehlt oder ist ein Link: " + directory) + env = services["backend"].get("environment", {}) + require(env.get("FEDEO_FILE_BACKEND", "s3") == "s3" and not env.get("INFISICAL_CLIENT_ID"), + "Externe Speicher/Secret-Provider benötigen einen eigenen Adapter.") + require(env.get("S3_ENDPOINT", "").rstrip("/") == "http://minio:9000", "Abweichender S3-Endpunkt.") + require(all(env.get(k) for k in ("S3_BUCKET", "S3_REGION", "S3_ACCESS_KEY", "S3_SECRET_KEY")), "S3-Konfiguration unvollständig.") + require(re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", env["S3_BUCKET"]), "Ungültiger Bucketname.") + require(env["S3_ACCESS_KEY"] == store["environment"]["MINIO_ROOT_USER"] and + env["S3_SECRET_KEY"] == store["environment"]["MINIO_ROOT_PASSWORD"], + "Getrennte MinIO-Benutzer vorhanden; deren Rechte zunächst vollständig inventarisieren.") + db = urllib.parse.urlsplit(env.get("DATABASE_URL", "")) + db_env = services["db"].get("environment", {}) + require(db.hostname == "db" and db.port in (None, 5432) and + urllib.parse.unquote(db.path.lstrip("/")) == db_env.get("POSTGRES_DB") and + urllib.parse.unquote(db.username or "") == db_env.get("POSTGRES_USER") and + urllib.parse.unquote(db.password or "") == db_env.get("POSTGRES_PASSWORD"), + "DATABASE_URL stimmt nicht mit der lokalen Sicherung überein.") + return env + + +class Migration: + def __init__(self, root, tools_image=None): + os.umask(0o077) + self.engine = u.Updater(root) + self.root = self.engine.root + self.home = self.engine.home / "storage-migration" + require(not self.home.is_symlink(), "Migrationsverzeichnis darf kein Link sein.") + self.home.mkdir(mode=0o700, exist_ok=True) + self.runner = u.Runner(self.home / "commands.log") + self.state = u.read_json(self.home / "state.json", {}) + self.tools_image = tools_image or self.state.get("toolsImage") + require(not self.state or tools_image in (None, self.state.get("toolsImage")), "Image eines bestehenden Auftrags darf nicht ersetzt werden.") + self.weed = u.read_json(HERE / "storage/seaweedfs.json") + self.helper_name = "fedeo-storage-helper-" + hashlib.sha256(str(self.root).encode()).hexdigest()[:12] + self.restore_name = self.helper_name + "-restore" + + def docker(self, *args, **kwargs): + return self.runner.run(["docker", *args], **kwargs) + + def compose(self, file, *args, **kwargs): + return self.docker("compose", "--project-directory", str(self.root), "--env-file", str(self.root / ".env"), + "-p", self.state["project"], "-f", str(file), *args, **kwargs) + + def save(self, phase=None, **fields): + self.state.update(fields, updatedAt=u.now()) + if phase: + self.state["phase"] = phase + u.write_json(self.home / "state.json", self.state) + + def guard(self): + # Compatible with the already installed 3.3 updater: it refuses recoveryRequired. + # Kept after cutover until a provider-aware application updater is installed. + u.write_json(self.engine.home / "status.json", { + "status": "failed", "phase": "storage_migration", "job": str(self.home), + "startedAt": self.state["startedAt"], "updatedAt": u.now(), "recoveryRequired": True, + "maintenance": not self.state.get("writesReleased", False), "mutationStarted": True, + "error": "Speichermigration: selfhost-storage-migrate.py status prüfen. Herkömmliche Updates sind gesperrt."}) + + def fingerprint(self): + return {name: sha_file(self.root / name) for name in ("docker-compose.yml", ".env")} + + def inspect(self, container): + return json.loads(self.docker("inspect", container))[0] + + def wait_ready(self, container, timeout=180): + for _ in range(timeout): + state = self.inspect(container)["State"] + require(state.get("Running"), "Ein benötigter Container wurde beendet.") + if state.get("Health", {}).get("Status") in (None, "healthy"): + return + time.sleep(1) + raise Error("Container wurde innerhalb der Wartezeit nicht bereit.") + + def cleanup_helper(self): + ids = self.docker("ps", "-aq", "--filter", "name=^/" + self.helper_name + "$").split() + for container in ids: + require(self.inspect(container)["Config"].get("Labels", {}).get("io.fedeo.storage-job") == str(self.home), + "Fremder Container belegt den Hilfsnamen.") + self.docker("rm", "-f", container) + + def helper(self, *args, **kwargs): + deadline = time.monotonic() + 180 + while True: + try: + return self.helper_once(*args, **kwargs) + except TransientStorageError: + if time.monotonic() >= deadline: + raise + time.sleep(2) + + def helper_once(self, command, *, source=False, admin=False, endpoint="http://objectstore:9000", directory="copy", backend_sdk=False): + self.cleanup_helper() + config = u.read_json(self.home / "credentials.json") + job = {"bucket": config["bucket"], "directory": "/work", "result": "/work/result.json"} + if command == "contract": + job["foreignBucket"] = "fedeo-denied-" + secrets.token_hex(8) + if source: + job["source"] = config["source"] + job["target"] = {**config["admin" if admin else "target"], "endpoint": endpoint} + u.write_json(self.home / "job.json", job) + work = self.home / directory + work.mkdir(mode=0o700, exist_ok=True) + (work / "error.json").unlink(missing_ok=True) + runtime = [self.tools_image, command] + if backend_sdk: + runtime = ["--entrypoint", "node", "-e", "FEDEO_STORAGE_SDK_ROOT=" + self.state["backendWorkingDir"], + "-v", str(HERE / "storage") + ":/fedeo-check:ro", self.state["containers"]["backend"]["image"], + "/fedeo-check/cli.mjs", command] + try: + self.docker("run", "--rm", "--pull", "never", "--name", self.helper_name, + "--label", "io.fedeo.storage-job=" + str(self.home), "--network", self.state["network"], + "--user", str(os.getuid()) + ":" + str(os.getgid()), + "--read-only", "--cap-drop=ALL", "--security-opt", "no-new-privileges", "--tmpfs", "/tmp", + "-v", str(self.home / "job.json") + ":/config/job.json:ro", "-v", str(work) + ":/work", + *runtime, timeout=24 * 3600, output=self.home / "progress.jsonl") + return u.read_json(work / "result.json") + except Error: + detail = u.read_json(work / "error.json", {}) + if detail.get("status") in (500, 502, 503, 504) or detail.get("code") in ("ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EHOSTUNREACH"): + raise TransientStorageError("S3 noch nicht bereit; Wiederholung blieb erfolglos.") from None + raise + finally: + self.cleanup_helper() + + def plan(self): + require(not self.state, "Ein Plan existiert bereits; status prüfen oder vorhandenen Auftrag fortsetzen.") + require(not self.engine.status().get("recoveryRequired"), "Zuerst den offenen Update-Auftrag klären.") + require(self.tools_image, "--tools-image mit lokalem Hilfsimage angeben.") + require(all((self.root / n).is_file() and not (self.root / n).is_symlink() for n in (".env", "docker-compose.yml")), + "Reguläre .env und docker-compose.yml erforderlich.") + require(not any((self.root / n).exists() for n in ("docker-compose.override.yml", "docker-compose.override.yaml", "compose.yml", "compose.yaml")), + "Compose-Overrides zuerst in einem gesonderten Plan berücksichtigen.") + endpoint = self.docker("context", "inspect", "--format", "{{.Endpoints.docker.Host}}").strip() + require(os.environ.get("DOCKER_HOST", endpoint).startswith("unix://"), + "Nur ein lokaler Docker-Daemon wird unterstützt.") + info = json.loads(self.docker("info", "--format", "{{json .}}")) + require(info["OSType"] == "linux" and info["Architecture"] in ("amd64", "x86_64"), "Pilot benötigt Linux amd64.") + config = resolved(json.loads(self.docker("compose", "--project-directory", str(self.root), "--env-file", str(self.root / ".env"), + "-f", str(self.root / "docker-compose.yml"), "config", "--format", "json"))) + env = validate_config(config, self.root) + # Reuse the updater's strict legacy Matrix path validation; the services remain intact. + self.engine.current = config + self.engine.retired_services = sorted(set(config["services"]) & u.LEGACY_MATRIX_SERVICES) + self.engine.matrix_backup_paths = [] + self.engine.validate_legacy_matrix() + project = config["name"] + ids = self.docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split() + containers = [self.inspect(i) for i in ids] + by_service = {} + for c in containers: + labels = c["Config"].get("Labels", {}) + name = labels.get("com.docker.compose.service") + require(name in config["services"] and name not in by_service and labels.get("com.docker.compose.oneoff", "False").lower() != "true", + "Unbekannte oder doppelte Projektcontainer vorhanden.") + by_service[name] = c + require(all(by_service.get(n, {}).get("State", {}).get("Running") for n in ("db", "minio", "backend", "frontend", "traefik")), + "Die bestehende Instanz muss vor der Planung laufen.") + for name in ("minio", "backend", "db"): + actual = dict(v.split("=", 1) for v in by_service[name]["Config"]["Env"] if "=" in v) + require(all(actual.get(k) == str(v) for k, v in config["services"][name].get("environment", {}).items()), + "Laufender Container und Compose-Umgebung weichen ab: " + name) + network = config["networks"]["internal"]["name"] + peers = json.loads(self.docker("network", "inspect", network))[0].get("Containers", {}) + require(set(peers) <= {c["Id"] for c in containers}, "Weitere Container im internen Netzwerk; externe Schreiber zuerst klären.") + for name, target in (("minio", "/data"), ("db", "/var/lib/postgresql/data")): + mounts = [m for m in by_service[name]["Mounts"] if m["Destination"] == target] + require(len(mounts) == 1 and mounts[0]["Source"] == str(self.root / ("minio" if name == "minio" else "postgres")), + "Laufender Datenmount weicht ab: " + name) + actual_store = by_service["minio"] + require(not actual_store["HostConfig"].get("PortBindings") and set(actual_store["NetworkSettings"]["Networks"]) == {network}, + "Laufendes MinIO hat zusätzliche Netzwerkzugänge.") + require(len(actual_store["Mounts"]) == 1, "Laufendes MinIO hat zusätzliche Mounts.") + require(not any(p.is_symlink() for p in (self.root / "minio").rglob("*")), "Links in MinIO-Daten benötigen eigene Backup-Prüfung.") + tools = json.loads(self.docker("image", "inspect", self.tools_image))[0] + weed = json.loads(self.docker("image", "inspect", self.weed["image"]))[0] + self.tools_image = tools["Id"] + # Containerd may discard an untagged image index after a mutable build tag + # moves. Keep an independent immutable job reference throughout the pilot. + tools_tag = "fedeo-storage-tools:job-" + secrets.token_hex(12) + self.docker("image", "tag", self.tools_image, tools_tag) + self.state = {"schemaVersion": 1, "phase": "planning", "startedAt": u.now(), "project": project, + "network": network, "toolsImage": self.tools_image, "weedImage": weed["Id"], + "toolsRetainTag": tools_tag, + "version": env.get("FEDEO_VERSION"), "fingerprint": self.fingerprint(), "writesReleased": False, + "backendWorkingDir": by_service["backend"]["Config"].get("WorkingDir") or "/", + "matrixPaths": self.engine.matrix_backup_paths, "containers": {name: {"id": c["Id"], "image": c["Image"], + "running": c["State"]["Running"], "restart": c["HostConfig"]["RestartPolicy"], + "networks": c["NetworkSettings"]["Networks"]} for name, c in by_service.items()}} + # Planning errors must not leave a usable partial state file. + u.write_json(self.home / "original.json", config) + for name in ("docker-compose.yml", ".env"): + u.atomic(self.home / ("original" + name.replace("/", "-")), (self.root / name).read_bytes()) + u.write_json(self.home / "previous-update.json", u.read_json(self.engine.home / "status.json", {"status": "idle"})) + admin = {"accessKeyId": "fedeo-migration-" + secrets.token_hex(8), "secretAccessKey": secrets.token_urlsafe(36)} + app = {"accessKeyId": env["S3_ACCESS_KEY"], "secretAccessKey": env["S3_SECRET_KEY"]} + connection = {"region": env["S3_REGION"], "credentials": app} + u.write_json(self.home / "credentials.json", {"bucket": env["S3_BUCKET"], + "source": {**connection, "endpoint": env["S3_ENDPOINT"]}, "target": connection, + "admin": {**connection, "credentials": admin}}) + inventory = self.helper("inventory", source=True, directory="preflight") + validate_minio_listing(self.root, env["S3_BUCKET"], inventory) + physical = int(self.runner.run(["du", "-sb", str(self.root / "minio")]).split()[0]) + db_size = int(self.docker("exec", by_service["db"]["Id"], "sh", "-ec", + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT pg_database_size(current_database())"').strip()) + image_ids = sorted({c["Image"] for c in containers} | {tools["Id"], weed["Id"]}) + image_bytes = sum(i["Size"] for i in json.loads(self.docker("image", "inspect", *image_ids))) + matrix_bytes = sum(int(self.runner.run(["du", "-sb", str(self.root / p)]).split()[0]) for p in self.state["matrixPaths"]) + # Source backup + target + restored target + offline images + DB + spool + generous metadata reserve. + needed = physical + inventory["bytes"] * 3 + inventory["count"] * 256 * 1024 + db_size * 2 + image_bytes * 2 + matrix_bytes + 5 * GIB + require(shutil.disk_usage(self.root).free >= needed, f"Zu wenig freier Platz; mindestens {needed} Bytes erforderlich.") + self.state.update(imageIds=image_ids, requiredFreeBytes=needed, freeBytes=shutil.disk_usage(self.root).free, + count=inventory["count"], bytes=inventory["bytes"]) + require(not (self.root / "storage").exists(), "storage/ existiert bereits; kein leeres Migrationsziel.") + s3config = {"identities": [ + {"name": "migration-admin", "credentials": [{"accessKey": admin["accessKeyId"], "secretKey": admin["secretAccessKey"]}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "fedeo", "credentials": [{"accessKey": app["accessKeyId"], "secretKey": app["secretAccessKey"]}], + "actions": [action + ":" + env["S3_BUCKET"] for action in ("Read", "Write", "List")]} + ]} + u.write_json(self.home / "s3.json", s3config) + self.make_compose(config) + self.save("planned") + + def make_compose(self, original): + target = copy.deepcopy(original) + store = copy.deepcopy(self.weed) + store["image"] = self.state["weedImage"] + store.update(container_name="fedeo-objectstore", restart="no", networks={"internal": {}}, + user=str(os.getuid()) + ":" + str(os.getgid()), + environment={"FEDEO_S3_CONFIG_SHA256": sha_file(self.home / "s3.json")}, + volumes=[{"type": "bind", "source": str(self.root / "storage/seaweedfs"), "target": "/data"}, + {"type": "bind", "source": str(self.root / "storage/config"), "target": "/etc/fedeo", "read_only": True}]) + stage = {"name": self.state["project"], "services": {"objectstore": store}, + "networks": {"internal": {"external": True, "name": self.state["network"]}}} + u.write_json(self.home / "stage.json", literal(stage)) + del target["services"]["minio"] + del target["services"]["createbuckets"] + target["services"]["objectstore"] = copy.deepcopy(store) + target["services"]["objectstore"]["networks"]["internal"] = {"aliases": ["minio"]} + target["services"]["objectstore"]["restart"] = "unless-stopped" + dependencies = target["services"]["backend"].setdefault("depends_on", {}) + dependencies.pop("minio", None) + dependencies.pop("createbuckets", None) + dependencies["objectstore"] = {"condition": "service_healthy"} + # Keep exactly the running application images; never couple this to an application/DB update. + for name, c in self.state["containers"].items(): + if name in target["services"]: + target["services"][name]["image"] = c["image"] + target["services"]["backend"]["environment"]["FEDEO_RUN_MIGRATIONS"] = "false" + # SDK 3.962 streaming trailers leave aws-chunked in SeaweedFS object metadata. + # SigV4 remains active; the migration tool supplies an explicit SHA-256 checksum. + target["services"]["backend"]["environment"]["AWS_REQUEST_CHECKSUM_CALCULATION"] = "WHEN_REQUIRED" + u.write_json(self.home / "target.json", literal(target)) + # Fail before maintenance if Compose cannot parse the generated configurations. + self.compose(self.home / "stage.json", "config", "--quiet") + self.compose(self.home / "target.json", "config", "--quiet") + + def verify_original(self): + require(self.fingerprint() == self.state["fingerprint"], "Compose/.env wurde nach der Planung verändert.") + for name, c in self.state["containers"].items(): + actual = self.inspect(c["id"]) + require(actual["Image"] == c["image"], "Containeridentität verändert: " + name) + + def freeze(self): + self.save("freezing") + self.guard() + for name, c in self.state["containers"].items(): + if name != "db": + self.docker("update", "--restart=no", c["id"]) + ids = [c["id"] for name, c in self.state["containers"].items() if name != "db"] + if ids: + self.docker("stop", "-t", "60", *ids) + self.assert_frozen() + + def assert_frozen(self): + require(self.inspect(self.state["containers"]["minio"]["id"])["HostConfig"]["RestartPolicy"]["Name"] == "no", + "MinIO könnte automatisch wieder starten.") + for name, c in self.state["containers"].items(): + if name in ("db", "minio"): + continue + actual = self.inspect(c["id"]) + require(not actual["State"]["Running"] and actual["HostConfig"]["RestartPolicy"]["Name"] == "no", + "Schreibpause wurde aufgehoben: " + name) + known = {c["id"] for c in self.state["containers"].values()} + peers = json.loads(self.docker("network", "inspect", self.state["network"]))[0].get("Containers", {}) + for cid in set(peers) - known: + c = self.inspect(cid) + labels = c["Config"].get("Labels", {}) + target = (labels.get("com.docker.compose.project") == self.state["project"] and + labels.get("com.docker.compose.service") == "objectstore" and c["Image"] == self.state["weedImage"]) + own_helper = labels.get("io.fedeo.storage-job") == str(self.home) + require(target or own_helper, "Ein zusätzlicher Container hat sich mit dem Speichernetz verbunden.") + + def check_backup(self): + backup = self.home / "backup" + complete = u.read_json(backup / "complete.json", {}) + require(complete.get("sha256"), "Abgeschlossene Sicherung fehlt.") + for name, expected in complete["sha256"].items(): + require(Path(name).name == name and sha_file(backup / name) == expected, + "Backup-Prüfsumme stimmt nicht: " + name) + + def backup(self): + self.save("backing_up") + backup = self.home / "backup" + backup.mkdir(mode=0o700, exist_ok=True) + require(not self.inspect(self.state["containers"]["minio"]["id"])["State"]["Running"], "MinIO für kalte Sicherung stoppen.") + db = self.state["containers"]["db"]["id"] + self.docker("exec", db, "sh", "-ec", 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc', output=backup / "database.dump") + self.docker("exec", "-i", db, "pg_restore", "--list", input_file=backup / "database.dump", output=backup / "database.list") + self.docker("exec", db, "sh", "-ec", 'exec pg_dumpall -U "$POSTGRES_USER" --globals-only', output=backup / "globals.sql") + paths = ["minio", *self.state["matrixPaths"], "storage/config", ".env", "docker-compose.yml"] + if (self.root / "traefik/letsencrypt").is_dir(): + paths.append("traefik/letsencrypt") + self.runner.run(["tar", "-cpf", str(backup / "files.tar"), "-C", str(self.root), *paths], timeout=7200) + self.docker("image", "save", "-o", str(backup / "images.tar"), *self.state["imageIds"], timeout=7200) + u.write_json(backup / "complete.json", {"at": u.now(), "sha256": {p.name: sha_file(p) for p in backup.iterdir() if p.is_file() and p.name != "complete.json"}}) + self.save("backed_up") + + def migrate(self, writers_exclusive): + require(writers_exclusive, "--writers-exclusive bestätigt: keine externen S3-/DB-Schreiber und kein paralleles manuelles Compose.") + require(self.state.get("phase") in ("planned", "freezing", "backing_up", "backed_up", "copying", "verifying", "verified"), + "Migration kann in dieser Phase nicht gestartet werden.") + self.cleanup_helper() + self.verify_original() + require(shutil.disk_usage(self.root).free >= (self.state["requiredFreeBytes"] if self.state["phase"] == "planned" else GIB), + "Nicht genug freier Speicher für Migration/Fortsetzung.") + if self.state["phase"] in ("planned", "freezing", "backing_up"): + (self.root / "storage/config").mkdir(parents=True, mode=0o700, exist_ok=True) + (self.root / "storage/seaweedfs/master").mkdir(parents=True, mode=0o700, exist_ok=True) + u.atomic(self.root / "storage/config/s3.json", (self.home / "s3.json").read_bytes()) + u.atomic(self.root / "storage/config/start-seaweedfs.sh", (HERE / "storage/start-seaweedfs.sh").read_bytes()) + self.freeze() + self.docker("start", self.state["containers"]["minio"]["id"]) + self.wait_ready(self.state["containers"]["minio"]["id"]) + current = self.helper("inventory", source=True, directory="frozen-inventory") + extra = max(0, current["bytes"] - self.state["bytes"]) * 4 + max(0, current["count"] - self.state["count"]) * 256 * 1024 + require(shutil.disk_usage(self.root).free >= self.state["requiredFreeBytes"] + extra, + "Bestand seit Planung gewachsen; Speicherreserve reicht nicht mehr.") + self.save(count=current["count"], bytes=current["bytes"]) + self.docker("stop", "-t", "60", self.state["containers"]["minio"]["id"]) + validate_minio_listing(self.root, u.read_json(self.home / "credentials.json")["bucket"], current, exact=True) + self.backup() + self.assert_frozen() + self.guard() + self.check_backup() + self.docker("start", self.state["containers"]["minio"]["id"]) + self.wait_ready(self.state["containers"]["minio"]["id"]) + self.compose(self.home / "stage.json", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "180") + self.helper("init", admin=True, directory="init") + self.helper("contract", directory="contract") + self.helper("contract", directory="backend-contract", backend_sdk=True) + self.save("copying") + self.helper("copy", source=True) + self.assert_frozen() + self.save("verifying") + self.helper("verify", source=True) + self.assert_frozen() + self.save("verified") + + def restore_test(self): + # Start a separate container on newly extracted directories. Never count a + # successful archive listing or restart of the original directory as restore. + self.save("restore_testing") + restore = self.home / "restore-test" + if restore.exists(): + # A previous interrupted restore is retained; use a fresh directory. + restore = self.home / ("restore-test-" + secrets.token_hex(4)) + restore.mkdir(mode=0o700) + self.runner.run(["tar", "-xpf", str(self.home / "backup/seaweedfs.tar"), "-C", str(restore)], timeout=7200) + known = self.docker("ps", "-aq", "--filter", "name=^/" + self.restore_name + "$").split() + for cid in known: + require(self.inspect(cid)["Config"].get("Labels", {}).get("io.fedeo.storage-job") == str(self.home), "Fremder Restore-Container.") + self.docker("rm", "-f", cid) + target = u.read_json(self.home / "stage.json")["services"]["objectstore"] + try: + self.docker("run", "-d", "--pull", "never", "--name", self.restore_name, + "--label", "io.fedeo.storage-job=" + str(self.home), "--network", self.state["network"], + "--user", str(os.getuid()) + ":" + str(os.getgid()), + "-v", str(restore / "storage/seaweedfs") + ":/data", + "-v", str(restore / "storage/config") + ":/etc/fedeo:ro", + "-e", "FEDEO_S3_CONFIG_SHA256=" + sha_file(self.home / "s3.json"), + "--entrypoint", target["entrypoint"][0], self.state["weedImage"], + *target["entrypoint"][1:], *target["command"]) + self.helper("restore-verify", endpoint="http://" + self.restore_name + ":9000") + self.helper("contract", endpoint="http://" + self.restore_name + ":9000", directory="restore-contract") + self.save("cutover", restoreVerifiedAt=u.now(), seaweedBackupSha256=sha_file(self.home / "backup/seaweedfs.tar")) + finally: + self.docker("rm", "-f", self.restore_name) + + def activate(self): + require(self.state.get("phase") in ("verified", "cutover", "restore_testing"), "Erst Kopie und Prüfung abschließen.") + require(not self.state.get("writesReleased"), "Schreibfreigabe ist bereits dauerhaft erfolgt.") + self.cleanup_helper() + self.assert_frozen() + # A killed contract test can leave only its journalled probe keys behind. + # Bring the target up without the legacy alias, remove those exact keys, + # then take the cold backup. Never delete unknown target objects. + self.compose(self.home / "stage.json", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "180") + for directory in ("contract", "backend-contract", "acceptance", "backend-acceptance"): + if (self.home / directory / "probe.json").is_file(): + self.helper("cleanup-probe", directory=directory) + if self.state["phase"] == "verified": + self.verify_original() + self.helper("verify", source=True) + self.save("cutover") + self.guard() + self.check_backup() + old = self.state["containers"]["minio"]["id"] + self.docker("stop", "-t", "60", old) + validate_minio_listing(self.root, u.read_json(self.home / "credentials.json")["bucket"], + u.read_json(self.home / "copy/inventory.json"), exact=True) + if self.state["network"] in self.inspect(old)["NetworkSettings"]["Networks"]: + self.docker("network", "disconnect", self.state["network"], old) + self.compose(self.home / "stage.json", "stop", "-t", "60", "objectstore") + # A cold backup is required before serving any business write on the new provider. + self.runner.run(["tar", "-cpf", str(self.home / "backup/seaweedfs.tar"), "-C", str(self.root), "storage"], timeout=7200) + self.restore_test() + u.atomic(self.root / "docker-compose.yml", (self.home / "target.json").read_bytes()) + self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "--wait", "--wait-timeout", "180", "objectstore") + self.helper("restore-verify", endpoint="http://minio:9000") + self.helper("contract", endpoint="http://minio:9000", directory="acceptance") + self.helper("contract", endpoint="http://minio:9000", directory="backend-acceptance", backend_sdk=True) + # Persist before starting any backend/background process; no automatic rollback beyond here. + self.save("released", writesReleased=True, releasedAt=u.now(), provider="seaweedfs") + self.guard() + self.start_application() + + def start_application(self): + require(self.state.get("writesReleased"), "Schreibfreigabe fehlt.") + require(sha_file(self.root / "docker-compose.yml") == sha_file(self.home / "target.json"), "Aktive Compose-Datei verändert.") + names = [n for n, c in self.state["containers"].items() if c["running"] and n not in ("minio", "createbuckets", "traefik")] + self.compose(self.root / "docker-compose.yml", "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "240", *names) + self.compose(self.root / "docker-compose.yml", "up", "-d", "--no-deps", "--pull", "never", "traefik") + self.save("complete") + + def rollback(self): + require(self.state and not self.state.get("writesReleased"), "Nach Schreibfreigabe ist eine geprüfte Rückmigration erforderlich.") + require(sha_file(self.root / ".env") == self.state["fingerprint"][".env"], ".env verändert; Rückkehr manuell prüfen.") + require(sha_file(self.root / "docker-compose.yml") in {self.state["fingerprint"]["docker-compose.yml"], sha_file(self.home / "target.json")}, + "Compose-Datei außerhalb der Migration verändert; Rückkehr manuell prüfen.") + self.cleanup_helper() + for cid in self.docker("ps", "-aq", "--filter", "name=^/" + self.restore_name + "$").split(): + require(self.inspect(cid)["Config"].get("Labels", {}).get("io.fedeo.storage-job") == str(self.home), "Fremder Restore-Container.") + self.docker("rm", "-f", cid) + require(self.state["phase"] != "planning", "Unvollständiger Plan.") + self.save("rolling_back") + self.guard() + # May not exist when the failure occurred before the first target start. + ids = self.docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + self.state["project"], + "--filter", "label=com.docker.compose.service=objectstore").split() + for cid in ids: + self.docker("update", "--restart=no", cid) + self.docker("stop", "-t", "60", cid) + if self.state["network"] in self.inspect(cid)["NetworkSettings"]["Networks"]: + self.docker("network", "disconnect", self.state["network"], cid) + u.atomic(self.root / "docker-compose.yml", (self.home / "originaldocker-compose.yml").read_bytes()) + for name, c in self.state["containers"].items(): + actual = self.inspect(c["id"]) + if name == "minio" and self.state["network"] not in actual["NetworkSettings"]["Networks"]: + aliases = c["networks"][self.state["network"]].get("Aliases") or ["minio"] + args = [arg for alias in aliases for arg in ("--alias", alias)] + self.docker("network", "connect", *args, self.state["network"], c["id"]) + policy = c["restart"]["Name"] + if policy == "on-failure" and c["restart"].get("MaximumRetryCount"): + policy += ":" + str(c["restart"]["MaximumRetryCount"]) + self.docker("update", "--restart=" + policy, c["id"]) + for name in ["db", "minio", *[n for n in self.state["containers"] if n not in ("db", "minio", "traefik")], "traefik"]: + c = self.state["containers"].get(name) + if c and c["running"]: + self.docker("start", c["id"]) + self.wait_ready(c["id"]) + self.save("rolled_back") + u.write_json(self.engine.home / "status.json", u.read_json(self.home / "previous-update.json")) + + def public_status(self): + return {k: self.state[k] for k in ("phase", "version", "count", "bytes", "requiredFreeBytes", "freeBytes", + "writesReleased", "releasedAt", "updatedAt") if k in self.state} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["plan", "migrate", "activate", "resume-start", "rollback", "status"]) + parser.add_argument("--directory", default="/opt/fedeo") + parser.add_argument("--tools-image") + parser.add_argument("--writers-exclusive", action="store_true") + args = parser.parse_args() + migration = Migration(args.directory, args.tools_image) + try: + if args.command == "status": + print(json.dumps(migration.public_status(), indent=2)); return + with migration.engine.lock(): + if args.command == "plan": migration.plan() + elif args.command == "migrate": migration.migrate(args.writers_exclusive) + elif args.command == "activate": migration.activate() + elif args.command == "resume-start": migration.start_application() + elif args.command == "rollback": migration.rollback() + print(json.dumps(migration.public_status(), indent=2)) + except (Error, OSError, ValueError, KeyboardInterrupt) as error: + print("Migration angehalten: " + str(error) + "\nGeschütztes Protokoll: " + str(migration.home / "commands.log"), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/storage/.dockerignore b/scripts/storage/.dockerignore new file mode 100644 index 00000000..63b1b97f --- /dev/null +++ b/scripts/storage/.dockerignore @@ -0,0 +1,2 @@ +node_modules +tests diff --git a/scripts/storage/Dockerfile b/scripts/storage/Dockerfile new file mode 100644 index 00000000..04207718 --- /dev/null +++ b/scripts/storage/Dockerfile @@ -0,0 +1,6 @@ +FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 +WORKDIR /opt/fedeo-storage +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund +COPY s3.mjs cli.mjs ./ +ENTRYPOINT ["node", "/opt/fedeo-storage/cli.mjs"] diff --git a/scripts/storage/README.md b/scripts/storage/README.md new file mode 100644 index 00000000..3def70dc --- /dev/null +++ b/scripts/storage/README.md @@ -0,0 +1,44 @@ +# Speicherwerkzeug für den betreuten Pilotbetrieb + +AWS SDK `3.962.0` entspricht dem bei Entwicklung vorliegenden Backend-Lockfile. +Der konkrete Ablauf auf einer Instanz steht in +[`docs/minio-umstellung-pilot.md`](../../docs/minio-umstellung-pilot.md). + +Lokaler Test (Linux amd64, Docker, Node 22): + +```sh +npm ci --prefix scripts/storage +npm test --prefix scripts/storage +docker pull chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882 +docker build -f scripts/storage/tests/Dockerfile.minio -t fedeo-minio-fixture:2025-09-07 scripts/storage/tests +docker build -t fedeo-storage-tools:pilot scripts/storage +node scripts/storage/tests/integration.mjs +python3 -B scripts/storage/tests/host_integration.py +``` + +Der S3-Test verwendet ausschließlich neu angelegte Testcontainer mit künstlichen +Daten. Er prüft über 1.000 Objekte, Pagination, 256 MiB, leere Dateien, +Sonderzeichen, Metadaten, SigV4, 24-Stunden-Links, Streaming, verbotene Zugriffe, +beschädigte Zielobjekte, erneute Kopie und echten Restore in ein frisches +Verzeichnis. Die gesicherten Testdaten bleiben unter `/tmp/fedeo-storage-*` +zur Prüfung erhalten; die Testcontainer werden entfernt. + +Der Host-Test verwendet einen echten PostgreSQL-/MinIO-/SeaweedFS-Stack und +künstliche Anwendungscontainer. Er ersetzt keine fachliche Abnahme von FEDEO +(PDF, WebDAV, Exporte, Mandantenrechte) auf der konkreten Instanz. + +Die erste Übernahme ist auf 200.000 Objekte und 256 MiB pro Objekt begrenzt. +Das größte Objekt wird auf Platte gespult; RAM-Verbrauch wächst nicht mit dessen +Größe. Multipart-Übernahmen und Sonderkonfigurationen werden abgelehnt. + +SeaweedFS 4.47 bewahrt bei SDK-Streaming mit automatischen Prüfsummen teilweise +`Content-Encoding: aws-chunked` auf. Der Kopierer setzt einen vorher berechneten +SHA-256-Checksum-Header. Die Pilot-Anwendung erhält +`AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED`; der Vertragstest prüft diese +Einstellung mit leeren und nicht leeren Streams. SigV4 bleibt aktiv. + +SeaweedFS-Herkunft: [Release 4.47](https://github.com/seaweedfs/seaweedfs/releases/tag/4.47). +Das isolierte MinIO-Testimage verwendet das offizielle, per SHA-256 verifizierte +[Release-Binary](https://github.com/minio/minio/releases/tag/RELEASE.2025-09-07T16-13-09Z), +da die früheren Containerreferenzen nicht mehr abrufbar sind. Es wird nicht im +Pilotpaket mitgeliefert und nicht über Kundendaten gestartet. diff --git a/scripts/storage/build-pilot.sh b/scripts/storage/build-pilot.sh new file mode 100644 index 00000000..5318a33c --- /dev/null +++ b/scripts/storage/build-pilot.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +storage_repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +storage_output="${1:-$storage_repo/.release/storage-pilot}" +test ! -e "$storage_output" +mkdir -p "$storage_output/scripts/storage" "$storage_output/docs" +storage_weed="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["image"])' "$storage_repo/scripts/storage/seaweedfs.json")" +if ! docker image inspect "$storage_weed" >/dev/null 2>&1; then docker pull "$storage_weed"; fi +docker build -t fedeo-storage-tools:pilot "$storage_repo/scripts/storage" +storage_tools_id="$(docker image inspect fedeo-storage-tools:pilot --format '{{.Id}}')" +storage_weed_id="$(docker image inspect "$storage_weed" --format '{{.Id}}')" +storage_tools="fedeo-storage-tools:pilot-${storage_tools_id#sha256:}" +storage_weed_local="fedeo-seaweedfs:pilot-${storage_weed_id#sha256:}" +docker image tag "$storage_tools_id" "$storage_tools" +docker image tag "$storage_weed_id" "$storage_weed_local" +cp "$storage_repo/scripts/selfhost-storage-migrate.py" "$storage_repo/scripts/selfhost-update.py" "$storage_output/scripts/" +cp "$storage_repo/scripts/storage/"{s3.mjs,cli.mjs,seaweedfs.json,start-seaweedfs.sh,package.json,package-lock.json} "$storage_output/scripts/storage/" +cp "$storage_repo/scripts/storage/pilot.py" "$storage_output/pilot.py" +cp "$storage_repo/docs/minio-umstellung-pilot.md" "$storage_output/docs/" +docker image save -o "$storage_output/images.tar" "$storage_tools" "$storage_weed_local" +python3 - "$storage_output" "$storage_tools" "$storage_weed_local" "$storage_weed" <<'PY' +import hashlib,json,pathlib,sys +root=pathlib.Path(sys.argv[1]).resolve() +spec=root/'scripts/storage/seaweedfs.json' +config=json.loads(spec.read_text()); config['image']=sys.argv[3] +spec.write_text(json.dumps(config,indent=2)+'\n') +checksums={} +for p in sorted(root.rglob('*')): + if p.is_file(): + h=hashlib.sha256() + with p.open('rb') as f: + for chunk in iter(lambda:f.read(1024*1024), b''): h.update(chunk) + checksums[str(p.relative_to(root))]=h.hexdigest() +(root/'bundle.json').write_text(json.dumps({'schemaVersion':1,'toolsImage':sys.argv[2], + 'seaweedImage':sys.argv[3],'upstreamSeaweedImage':sys.argv[4],'sha256':checksums},indent=2)+'\n') +PY +tar -czf "$storage_output.tar.gz" -C "$(dirname "$storage_output")" "$(basename "$storage_output")" +(cd "$(dirname "$storage_output")" && sha256sum "$(basename "$storage_output").tar.gz" > "$(basename "$storage_output").tar.gz.sha256") +printf 'Pilotpaket: %s.tar.gz\n' "$storage_output" diff --git a/scripts/storage/cli.mjs b/scripts/storage/cli.mjs new file mode 100644 index 00000000..bde55fa4 --- /dev/null +++ b/scripts/storage/cli.mjs @@ -0,0 +1,39 @@ +import { readFile } from 'node:fs/promises'; +import { client, call, inventory, transfer, contract, verifyManifest, atomicJson, cleanupProbe } from './s3.mjs'; + +// Secrets enter through a root-only mounted JSON file, never through argv or logs. +const [command, configPath = '/config/job.json'] = process.argv.slice(2); +if (command === 'contract') process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED'; +let source, target; +try { + const config = JSON.parse(await readFile(configPath, 'utf8')); + source = config.source && client(config.source); + target = config.target && client(config.target); + const bucket = config.bucket; + let result; + if (command === 'inventory') result = await inventory(source, bucket); + else if (command === 'copy' || command === 'verify') { + result = await transfer(source, target, bucket, config.directory || '/work', { verifyOnly: command === 'verify' }); + result = { ...result, objects: undefined }; + } else if (command === 'init') { + try { await call(target, 'HeadBucket', { Bucket: bucket }); } + catch (error) { + if (error.$metadata?.httpStatusCode !== 404) throw error; + await call(target, 'CreateBucket', { Bucket: bucket }); + } + result = { bucket, initialized: true }; + } else if (command === 'contract') result = await contract(target, config.target, bucket, { ...config, probePath: '/work/probe.json' }); + else if (command === 'cleanup-probe') { await cleanupProbe(target, bucket, '/work/probe.json'); result = { cleaned: true }; } + else if (command === 'restore-verify') { + result = await verifyManifest(target, bucket, JSON.parse(await readFile(config.manifest || '/work/verified.json', 'utf8'))); + } else throw new Error('Unbekannter Befehl.'); + await atomicJson(config.result || '/work/result.json', result); + console.log(JSON.stringify(command === 'inventory' ? { count: result.count, bytes: result.bytes } : result)); +} catch (error) { + // SDK error messages/URLs may contain identifiers; keep public output bounded. + const detail = { error: error.name, code: error.code, status: error.$metadata?.httpStatusCode, + message: error.$metadata ? 'S3-Prüfung fehlgeschlagen; Zugriff/Konfiguration prüfen.' : error.message }; + try { await atomicJson('/work/error.json', detail); } catch { /* Preserve the original error. */ } + console.error(JSON.stringify(detail)); + process.exitCode = 1; +} finally { source?.destroy(); target?.destroy(); } diff --git a/scripts/storage/package-lock.json b/scripts/storage/package-lock.json new file mode 100644 index 00000000..cc12dfca --- /dev/null +++ b/scripts/storage/package-lock.json @@ -0,0 +1,1740 @@ +{ + "name": "@fedeo/storage-migration", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@fedeo/storage-migration", + "version": "1.0.0", + "dependencies": { + "@aws-sdk/client-s3": "3.962.0", + "@aws-sdk/s3-request-presigner": "3.962.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/crc32c": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", + "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha1-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.962.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.962.0.tgz", + "integrity": "sha512-I2/1McBZCcM3PfM4ck8D6gnZR3K7+yl1fGkwTq/3ThEn9tdLjNwcdgTbPfxfX6LoecLrH9Ekoo+D9nmQ0T261w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha1-browser": "5.2.0", + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.957.0", + "@aws-sdk/credential-provider-node": "3.962.0", + "@aws-sdk/middleware-bucket-endpoint": "3.957.0", + "@aws-sdk/middleware-expect-continue": "3.957.0", + "@aws-sdk/middleware-flexible-checksums": "3.957.0", + "@aws-sdk/middleware-host-header": "3.957.0", + "@aws-sdk/middleware-location-constraint": "3.957.0", + "@aws-sdk/middleware-logger": "3.957.0", + "@aws-sdk/middleware-recursion-detection": "3.957.0", + "@aws-sdk/middleware-sdk-s3": "3.957.0", + "@aws-sdk/middleware-ssec": "3.957.0", + "@aws-sdk/middleware-user-agent": "3.957.0", + "@aws-sdk/region-config-resolver": "3.957.0", + "@aws-sdk/signature-v4-multi-region": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-endpoints": "3.957.0", + "@aws-sdk/util-user-agent-browser": "3.957.0", + "@aws-sdk/util-user-agent-node": "3.957.0", + "@smithy/config-resolver": "^4.4.5", + "@smithy/core": "^3.20.0", + "@smithy/eventstream-serde-browser": "^4.2.7", + "@smithy/eventstream-serde-config-resolver": "^4.3.7", + "@smithy/eventstream-serde-node": "^4.2.7", + "@smithy/fetch-http-handler": "^5.3.8", + "@smithy/hash-blob-browser": "^4.2.8", + "@smithy/hash-node": "^4.2.7", + "@smithy/hash-stream-node": "^4.2.7", + "@smithy/invalid-dependency": "^4.2.7", + "@smithy/md5-js": "^4.2.7", + "@smithy/middleware-content-length": "^4.2.7", + "@smithy/middleware-endpoint": "^4.4.1", + "@smithy/middleware-retry": "^4.4.17", + "@smithy/middleware-serde": "^4.2.8", + "@smithy/middleware-stack": "^4.2.7", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/node-http-handler": "^4.4.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/url-parser": "^4.2.7", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.16", + "@smithy/util-defaults-mode-node": "^4.2.19", + "@smithy/util-endpoints": "^3.2.7", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-retry": "^4.2.7", + "@smithy/util-stream": "^4.5.8", + "@smithy/util-utf8": "^4.2.0", + "@smithy/util-waiter": "^4.2.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-sso": { + "version": "3.958.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.958.0.tgz", + "integrity": "sha512-6qNCIeaMzKzfqasy2nNRuYnMuaMebCcCPP4J2CVGkA8QYMbIVKPlkn9bpB20Vxe6H/r3jtCCLQaOJjVTx/6dXg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.957.0", + "@aws-sdk/middleware-host-header": "3.957.0", + "@aws-sdk/middleware-logger": "3.957.0", + "@aws-sdk/middleware-recursion-detection": "3.957.0", + "@aws-sdk/middleware-user-agent": "3.957.0", + "@aws-sdk/region-config-resolver": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-endpoints": "3.957.0", + "@aws-sdk/util-user-agent-browser": "3.957.0", + "@aws-sdk/util-user-agent-node": "3.957.0", + "@smithy/config-resolver": "^4.4.5", + "@smithy/core": "^3.20.0", + "@smithy/fetch-http-handler": "^5.3.8", + "@smithy/hash-node": "^4.2.7", + "@smithy/invalid-dependency": "^4.2.7", + "@smithy/middleware-content-length": "^4.2.7", + "@smithy/middleware-endpoint": "^4.4.1", + "@smithy/middleware-retry": "^4.4.17", + "@smithy/middleware-serde": "^4.2.8", + "@smithy/middleware-stack": "^4.2.7", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/node-http-handler": "^4.4.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/url-parser": "^4.2.7", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.16", + "@smithy/util-defaults-mode-node": "^4.2.19", + "@smithy/util-endpoints": "^3.2.7", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-retry": "^4.2.7", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.957.0.tgz", + "integrity": "sha512-DrZgDnF1lQZv75a52nFWs6MExihJF2GZB6ETZRqr6jMwhrk2kbJPUtvgbifwcL7AYmVqHQDJBrR/MqkwwFCpiw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@aws-sdk/xml-builder": "3.957.0", + "@smithy/core": "^3.20.0", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/property-provider": "^4.2.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/signature-v4": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/crc64-nvme": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.957.0.tgz", + "integrity": "sha512-qSwSfI+qBU9HDsd6/4fM9faCxYJx2yDuHtj+NVOQ6XYDWQzFab/hUdwuKZ77Pi6goLF1pBZhJ2azaC2w7LbnTA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.957.0.tgz", + "integrity": "sha512-475mkhGaWCr+Z52fOOVb/q2VHuNvqEDixlYIkeaO6xJ6t9qR0wpLt4hOQaR6zR1wfZV0SlE7d8RErdYq/PByog==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@smithy/property-provider": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.957.0.tgz", + "integrity": "sha512-8dS55QHRxXgJlHkEYaCGZIhieCs9NU1HU1BcqQ4RfUdSsfRdxxktqUKgCnBnOOn0oD3PPA8cQOCAVgIyRb3Rfw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@smithy/fetch-http-handler": "^5.3.8", + "@smithy/node-http-handler": "^4.4.7", + "@smithy/property-provider": "^4.2.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/util-stream": "^4.5.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.962.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.962.0.tgz", + "integrity": "sha512-h0kVnXLW2d3nxbcrR/Pfg3W/+YoCguasWz7/3nYzVqmdKarGrpJzaFdoZtLgvDSZ8VgWUC4lWOTcsDMV0UNqUQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/credential-provider-env": "3.957.0", + "@aws-sdk/credential-provider-http": "3.957.0", + "@aws-sdk/credential-provider-login": "3.962.0", + "@aws-sdk/credential-provider-process": "3.957.0", + "@aws-sdk/credential-provider-sso": "3.958.0", + "@aws-sdk/credential-provider-web-identity": "3.958.0", + "@aws-sdk/nested-clients": "3.958.0", + "@aws-sdk/types": "3.957.0", + "@smithy/credential-provider-imds": "^4.2.7", + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.962.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.962.0.tgz", + "integrity": "sha512-kHYH6Av2UifG3mPkpPUNRh/PuX6adaAcpmsclJdHdxlixMCRdh8GNeEihq480DC0GmfqdpoSf1w2CLmLLPIS6w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/nested-clients": "3.958.0", + "@aws-sdk/types": "3.957.0", + "@smithy/property-provider": "^4.2.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.962.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.962.0.tgz", + "integrity": "sha512-CS78NsWRxLa+nWqeWBEYMZTLacMFIXs1C5WJuM9kD05LLiWL32ksljoPsvNN24Bc7rCSQIIMx/U3KGvkDVZMVg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.957.0", + "@aws-sdk/credential-provider-http": "3.957.0", + "@aws-sdk/credential-provider-ini": "3.962.0", + "@aws-sdk/credential-provider-process": "3.957.0", + "@aws-sdk/credential-provider-sso": "3.958.0", + "@aws-sdk/credential-provider-web-identity": "3.958.0", + "@aws-sdk/types": "3.957.0", + "@smithy/credential-provider-imds": "^4.2.7", + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.957.0.tgz", + "integrity": "sha512-/KIz9kadwbeLy6SKvT79W81Y+hb/8LMDyeloA2zhouE28hmne+hLn0wNCQXAAupFFlYOAtZR2NTBs7HBAReJlg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.958.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.958.0.tgz", + "integrity": "sha512-CBYHJ5ufp8HC4q+o7IJejCUctJXWaksgpmoFpXerbjAso7/Fg7LLUu9inXVOxlHKLlvYekDXjIUBXDJS2WYdgg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-sso": "3.958.0", + "@aws-sdk/core": "3.957.0", + "@aws-sdk/token-providers": "3.958.0", + "@aws-sdk/types": "3.957.0", + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.958.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.958.0.tgz", + "integrity": "sha512-dgnvwjMq5Y66WozzUzxNkCFap+umHUtqMMKlr8z/vl9NYMLem/WUbWNpFFOVFWquXikc+ewtpBMR4KEDXfZ+KA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/nested-clients": "3.958.0", + "@aws-sdk/types": "3.957.0", + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.957.0.tgz", + "integrity": "sha512-iczcn/QRIBSpvsdAS/rbzmoBpleX1JBjXvCynMbDceVLBIcVrwT1hXECrhtIC2cjh4HaLo9ClAbiOiWuqt+6MA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-arn-parser": "3.957.0", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "@smithy/util-config-provider": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.957.0.tgz", + "integrity": "sha512-AlbK3OeVNwZZil0wlClgeI/ISlOt/SPUxBsIns876IFaVu/Pj3DgImnYhpcJuFRek4r4XM51xzIaGQXM6GDHGg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.957.0.tgz", + "integrity": "sha512-iJpeVR5V8se1hl2pt+k8bF/e9JO4KWgPCMjg8BtRspNtKIUGy7j6msYvbDixaKZaF2Veg9+HoYcOhwnZumjXSA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@aws-crypto/crc32c": "5.2.0", + "@aws-crypto/util": "5.2.0", + "@aws-sdk/core": "3.957.0", + "@aws-sdk/crc64-nvme": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@smithy/is-array-buffer": "^4.2.0", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-stream": "^4.5.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-host-header": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.957.0.tgz", + "integrity": "sha512-BBgKawVyfQZglEkNTuBBdC3azlyqNXsvvN4jPkWAiNYcY0x1BasaJFl+7u/HisfULstryweJq/dAvIZIxzlZaA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-location-constraint": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.957.0.tgz", + "integrity": "sha512-y8/W7TOQpmDJg/fPYlqAhwA4+I15LrS7TwgUEoxogtkD8gfur9wFMRLT8LCyc9o4NMEcAnK50hSb4+wB0qv6tQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-logger": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.957.0.tgz", + "integrity": "sha512-w1qfKrSKHf9b5a8O76yQ1t69u6NWuBjr5kBX+jRWFx/5mu6RLpqERXRpVJxfosbep7k3B+DSB5tZMZ82GKcJtQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.957.0.tgz", + "integrity": "sha512-D2H/WoxhAZNYX+IjkKTdOhOkWQaK0jjJrDBj56hKjU5c9ltQiaX/1PqJ4dfjHntEshJfu0w+E6XJ+/6A6ILBBA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.957.0.tgz", + "integrity": "sha512-5B2qY2nR2LYpxoQP0xUum5A1UNvH2JQpLHDH1nWFNF/XetV7ipFHksMxPNhtJJ6ARaWhQIDXfOUj0jcnkJxXUg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-arn-parser": "3.957.0", + "@smithy/core": "^3.20.0", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/signature-v4": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/util-config-provider": "^4.2.0", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-stream": "^4.5.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-ssec": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.957.0.tgz", + "integrity": "sha512-qwkmrK0lizdjNt5qxl4tHYfASh8DFpHXM1iDVo+qHe+zuslfMqQEGRkzxS8tJq/I+8F0c6v3IKOveKJAfIvfqQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.957.0.tgz", + "integrity": "sha512-50vcHu96XakQnIvlKJ1UoltrFODjsq2KvtTgHiPFteUS884lQnK5VC/8xd1Msz/1ONpLMzdCVproCQqhDTtMPQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-endpoints": "3.957.0", + "@smithy/core": "^3.20.0", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.958.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.958.0.tgz", + "integrity": "sha512-/KuCcS8b5TpQXkYOrPLYytrgxBhv81+5pChkOlhegbeHttjM69pyUpQVJqyfDM/A7wPLnDrzCAnk4zaAOkY0Nw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.957.0", + "@aws-sdk/middleware-host-header": "3.957.0", + "@aws-sdk/middleware-logger": "3.957.0", + "@aws-sdk/middleware-recursion-detection": "3.957.0", + "@aws-sdk/middleware-user-agent": "3.957.0", + "@aws-sdk/region-config-resolver": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-endpoints": "3.957.0", + "@aws-sdk/util-user-agent-browser": "3.957.0", + "@aws-sdk/util-user-agent-node": "3.957.0", + "@smithy/config-resolver": "^4.4.5", + "@smithy/core": "^3.20.0", + "@smithy/fetch-http-handler": "^5.3.8", + "@smithy/hash-node": "^4.2.7", + "@smithy/invalid-dependency": "^4.2.7", + "@smithy/middleware-content-length": "^4.2.7", + "@smithy/middleware-endpoint": "^4.4.1", + "@smithy/middleware-retry": "^4.4.17", + "@smithy/middleware-serde": "^4.2.8", + "@smithy/middleware-stack": "^4.2.7", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/node-http-handler": "^4.4.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/url-parser": "^4.2.7", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.16", + "@smithy/util-defaults-mode-node": "^4.2.19", + "@smithy/util-endpoints": "^3.2.7", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-retry": "^4.2.7", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/region-config-resolver": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.957.0.tgz", + "integrity": "sha512-V8iY3blh8l2iaOqXWW88HbkY5jDoWjH56jonprG/cpyqqCnprvpMUZWPWYJoI8rHRf2bqzZeql1slxG6EnKI7A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/config-resolver": "^4.4.5", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/s3-request-presigner": { + "version": "3.962.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.962.0.tgz", + "integrity": "sha512-tyxsGfLY4NSohLrJsFGXbE3j8jguWK+hdGaUQSD1gJPvmC0B82qOyJ7WBIJLWgTabU3fiF/I9EGXjzR2rKr8jQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/signature-v4-multi-region": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@aws-sdk/util-format-url": "3.957.0", + "@smithy/middleware-endpoint": "^4.4.1", + "@smithy/protocol-http": "^5.3.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.957.0.tgz", + "integrity": "sha512-t6UfP1xMUigMMzHcb7vaZcjv7dA2DQkk9C/OAP1dKyrE0vb4lFGDaTApi17GN6Km9zFxJthEMUbBc7DL0hq1Bg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-sdk-s3": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@smithy/protocol-http": "^5.3.7", + "@smithy/signature-v4": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.958.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.958.0.tgz", + "integrity": "sha512-UCj7lQXODduD1myNJQkV+LYcGYJ9iiMggR8ow8Hva1g3A/Na5imNXzz6O67k7DAee0TYpy+gkNw+SizC6min8Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.957.0", + "@aws-sdk/nested-clients": "3.958.0", + "@aws-sdk/types": "3.957.0", + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.957.0.tgz", + "integrity": "sha512-wzWC2Nrt859ABk6UCAVY/WYEbAd7FjkdrQL6m24+tfmWYDNRByTJ9uOgU/kw9zqLCAwb//CPvrJdhqjTznWXAg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/util-arn-parser": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.957.0.tgz", + "integrity": "sha512-Aj6m+AyrhWyg8YQ4LDPg2/gIfGHCEcoQdBt5DeSFogN5k9mmJPOJ+IAmNSWmWRjpOxEy6eY813RNDI6qS97M0g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/util-endpoints": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.957.0.tgz", + "integrity": "sha512-xwF9K24mZSxcxKS3UKQFeX/dPYkEps9wF1b+MGON7EvnbcucrJGyQyK1v1xFPn1aqXkBTFi+SZaMRx5E5YCVFw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/types": "^4.11.0", + "@smithy/url-parser": "^4.2.7", + "@smithy/util-endpoints": "^3.2.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/util-format-url": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.957.0.tgz", + "integrity": "sha512-Yyo/tlc0iGFGTPPkuxub1uRAv6XrnVnvSNjslZh5jIYA8GZoeEFPgJa3Qdu0GUS/YwoK8GOLnnaL9h/eH5LDJQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/querystring-builder": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.957.0.tgz", + "integrity": "sha512-nhmgKHnNV9K+i9daumaIz8JTLsIIML9PE/HUks5liyrjUzenjW/aHoc7WJ9/Td/gPZtayxFnXQSJRb/fDlBuJw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.957.0.tgz", + "integrity": "sha512-exueuwxef0lUJRnGaVkNSC674eAiWU07ORhxBnevFFZEKisln+09Qrtw823iyv5I1N8T+wKfh95xvtWQrNKNQw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.957.0", + "@smithy/types": "^4.11.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.957.0.tgz", + "integrity": "sha512-ycbYCwqXk4gJGp0Oxkzf2KBeeGBdTxz559D41NJP8FlzSej1Gh7Rk40Zo6AyTfsNWkrl/kVi1t937OIzC5t+9Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.957.0", + "@aws-sdk/types": "3.957.0", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.957.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.957.0.tgz", + "integrity": "sha512-Ai5iiQqS8kJ5PjzMhWcLKN0G2yasAkvpnPlq2EnqlIMdB48HsizElt62qcktdxp4neRMyGkFq4NzgmDbXnhRiA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "fast-xml-parser": "5.2.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder/node_modules/fast-xml-parser": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.5.tgz", + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.2.tgz", + "integrity": "sha512-C0NBLsIqzDIae8HFw9YIrIBsbc0xTiOtt7fAukGPnqQ/+zZNaq+4jhuccltK0QuWHBnNm/a6kLIRA6GFiM10eg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/abort-controller": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.7.tgz", + "integrity": "sha512-rzMY6CaKx2qxrbYbqjXWS0plqEy7LOdKHS0bg4ixJ6aoGDPNUcLWk/FRNuCILh7GKLG9TFUXYYeQQldMBBwuyw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-5.2.0.tgz", + "integrity": "sha512-WmU0TnhEAJLWvfSeMxBNe5xtbselEO8+4wG0NtZeL8oR21WgH1xiO37El+/Y+H/Ie4SCwBy3MxYWmOYaGgZueA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader-native": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-4.2.1.tgz", + "integrity": "sha512-lX9Ay+6LisTfpLid2zZtIhSEjHMZoAR5hHCR4H7tBz/Zkfr5ea8RcQ7Tk4mi0P76p4cN+Btz16Ffno7YHpKXnQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-base64": "^4.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/config-resolver": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.5.tgz", + "integrity": "sha512-HAGoUAFYsUkoSckuKbCPayECeMim8pOu+yLy1zOxt1sifzEbrsRpYa+mKcMdiHKMeiqOibyPG0sFJnmaV/OGEg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/node-config-provider": "^4.3.7", + "@smithy/types": "^4.11.0", + "@smithy/util-config-provider": "^4.2.0", + "@smithy/util-endpoints": "^3.2.7", + "@smithy/util-middleware": "^4.2.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/core": { + "version": "3.20.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.20.0.tgz", + "integrity": "sha512-WsSHCPq/neD5G/MkK4csLI5Y5Pkd9c1NMfpYEKeghSGaD4Ja1qLIohRQf2D5c1Uy5aXp76DeKHkzWZ9KAlHroQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/middleware-serde": "^4.2.8", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-stream": "^4.5.8", + "@smithy/util-utf8": "^4.2.0", + "@smithy/uuid": "^1.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.7.tgz", + "integrity": "sha512-CmduWdCiILCRNbQWFR0OcZlUPVtyE49Sr8yYL0rZQ4D/wKxiNzBNS/YHemvnbkIWj623fplgkexUd/c9CAKdoA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/node-config-provider": "^4.3.7", + "@smithy/property-provider": "^4.2.7", + "@smithy/types": "^4.11.0", + "@smithy/url-parser": "^4.2.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-codec": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.7.tgz", + "integrity": "sha512-DrpkEoM3j9cBBWhufqBwnbbn+3nf1N9FP6xuVJ+e220jbactKuQgaZwjwP5CP1t+O94brm2JgVMD2atMGX3xIQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.11.0", + "@smithy/util-hex-encoding": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-browser": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.7.tgz", + "integrity": "sha512-ujzPk8seYoDBmABDE5YqlhQZAXLOrtxtJLrbhHMKjBoG5b4dK4i6/mEU+6/7yXIAkqOO8sJ6YxZl+h0QQ1IJ7g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/eventstream-serde-universal": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-config-resolver": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.7.tgz", + "integrity": "sha512-x7BtAiIPSaNaWuzm24Q/mtSkv+BrISO/fmheiJ39PKRNH3RmH2Hph/bUKSOBOBC9unqfIYDhKTHwpyZycLGPVQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-node": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.7.tgz", + "integrity": "sha512-roySCtHC5+pQq5lK4be1fZ/WR6s/AxnPaLfCODIPArtN2du8s5Ot4mKVK3pPtijL/L654ws592JHJ1PbZFF6+A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/eventstream-serde-universal": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.7.tgz", + "integrity": "sha512-QVD+g3+icFkThoy4r8wVFZMsIP08taHVKjE6Jpmz8h5CgX/kk6pTODq5cht0OMtcapUx+xrPzUTQdA+TmO0m1g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/eventstream-codec": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.3.8", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.8.tgz", + "integrity": "sha512-h/Fi+o7mti4n8wx1SR6UHWLaakwHRx29sizvp8OOm7iqwKGFneT06GCSFhml6Bha5BT6ot5pj3CYZnCHhGC2Rg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/protocol-http": "^5.3.7", + "@smithy/querystring-builder": "^4.2.7", + "@smithy/types": "^4.11.0", + "@smithy/util-base64": "^4.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/hash-blob-browser": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.8.tgz", + "integrity": "sha512-07InZontqsM1ggTCPSRgI7d8DirqRrnpL7nIACT4PW0AWrgDiHhjGZzbAE5UtRSiU0NISGUYe7/rri9ZeWyDpw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/chunked-blob-reader": "^5.2.0", + "@smithy/chunked-blob-reader-native": "^4.2.1", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/hash-node": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.7.tgz", + "integrity": "sha512-PU/JWLTBCV1c8FtB8tEFnY4eV1tSfBc7bDBADHfn1K+uRbPgSJ9jnJp0hyjiFN2PMdPzxsf1Fdu0eo9fJ760Xw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/hash-stream-node": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.7.tgz", + "integrity": "sha512-ZQVoAwNYnFMIbd4DUc517HuwNelJUY6YOzwqrbcAgCnVn+79/OK7UjwA93SPpdTOpKDVkLIzavWm/Ck7SmnDPQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/invalid-dependency": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.7.tgz", + "integrity": "sha512-ncvgCr9a15nPlkhIUx3CU4d7E7WEuVJOV7fS7nnK2hLtPK9tYRBkMHQbhXU1VvvKeBm/O0x26OEoBq+ngFpOEQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.2.0.tgz", + "integrity": "sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/md5-js": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.7.tgz", + "integrity": "sha512-Wv6JcUxtOLTnxvNjDnAiATUsk8gvA6EeS8zzHig07dotpByYsLot+m0AaQEniUBjx97AC41MQR4hW0baraD1Xw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/middleware-content-length": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.7.tgz", + "integrity": "sha512-GszfBfCcvt7kIbJ41LuNa5f0wvQCHhnGx/aDaZJCCT05Ld6x6U2s0xsc/0mBFONBZjQJp2U/0uSJ178OXOwbhg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/middleware-endpoint": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.1.tgz", + "integrity": "sha512-gpLspUAoe6f1M6H0u4cVuFzxZBrsGZmjx2O9SigurTx4PbntYa4AJ+o0G0oGm1L2oSX6oBhcGHwrfJHup2JnJg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.20.0", + "@smithy/middleware-serde": "^4.2.8", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "@smithy/url-parser": "^4.2.7", + "@smithy/util-middleware": "^4.2.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/middleware-retry": { + "version": "4.4.17", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.17.tgz", + "integrity": "sha512-MqbXK6Y9uq17h+4r0ogu/sBT6V/rdV+5NvYL7ZV444BKfQygYe8wAhDrVXagVebN6w2RE0Fm245l69mOsPGZzg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/node-config-provider": "^4.3.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/service-error-classification": "^4.2.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-retry": "^4.2.7", + "@smithy/uuid": "^1.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/middleware-serde": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.8.tgz", + "integrity": "sha512-8rDGYen5m5+NV9eHv9ry0sqm2gI6W7mc1VSFMtn6Igo25S507/HaOX9LTHAS2/J32VXD0xSzrY0H5FJtOMS4/w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/middleware-stack": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.7.tgz", + "integrity": "sha512-bsOT0rJ+HHlZd9crHoS37mt8qRRN/h9jRve1SXUhVbkRzu0QaNYZp1i1jha4n098tsvROjcwfLlfvcFuJSXEsw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-config-provider": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.7.tgz", + "integrity": "sha512-7r58wq8sdOcrwWe+klL9y3bc4GW1gnlfnFOuL7CXa7UzfhzhxKuzNdtqgzmTV+53lEp9NXh5hY/S4UgjLOzPfw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/property-provider": "^4.2.7", + "@smithy/shared-ini-file-loader": "^4.4.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.4.7", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.7.tgz", + "integrity": "sha512-NELpdmBOO6EpZtWgQiHjoShs1kmweaiNuETUpuup+cmm/xJYjT4eUjfhrXRP4jCOaAsS3c3yPsP3B+K+/fyPCQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/abort-controller": "^4.2.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/querystring-builder": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/property-provider": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.7.tgz", + "integrity": "sha512-jmNYKe9MGGPoSl/D7JDDs1C8b3dC8f/w78LbaVfoTtWy4xAd5dfjaFG9c9PWPihY4ggMQNQSMtzU77CNgAJwmA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/protocol-http": { + "version": "5.3.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.7.tgz", + "integrity": "sha512-1r07pb994I20dD/c2seaZhoCuNYm0rWrvBxhCQ70brNh11M5Ml2ew6qJVo0lclB3jMIXirD4s2XRXRe7QEi0xA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/querystring-builder": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.7.tgz", + "integrity": "sha512-eKONSywHZxK4tBxe2lXEysh8wbBdvDWiA+RIuaxZSgCMmA0zMgoDpGLJhnyj+c0leOQprVnXOmcB4m+W9Rw7sg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "@smithy/util-uri-escape": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/querystring-parser": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.7.tgz", + "integrity": "sha512-3X5ZvzUHmlSTHAXFlswrS6EGt8fMSIxX/c3Rm1Pni3+wYWB6cjGocmRIoqcQF9nU5OgGmL0u7l9m44tSUpfj9w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/service-error-classification": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.7.tgz", + "integrity": "sha512-YB7oCbukqEb2Dlh3340/8g8vNGbs/QsNNRms+gv3N2AtZz9/1vSBx6/6tpwQpZMEJFs7Uq8h4mmOn48ZZ72MkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.2.tgz", + "integrity": "sha512-M7iUUff/KwfNunmrgtqBfvZSzh3bmFgv/j/t1Y1dQ+8dNo34br1cqVEqy6v0mYEgi0DkGO7Xig0AnuOaEGVlcg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.3.7", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.7.tgz", + "integrity": "sha512-9oNUlqBlFZFOSdxgImA6X5GFuzE7V2H7VG/7E70cdLhidFbdtvxxt81EHgykGK5vq5D3FafH//X+Oy31j3CKOg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^4.2.0", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "@smithy/util-hex-encoding": "^4.2.0", + "@smithy/util-middleware": "^4.2.7", + "@smithy/util-uri-escape": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/smithy-client": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.10.2.tgz", + "integrity": "sha512-D5z79xQWpgrGpAHb054Fn2CCTQZpog7JELbVQ6XAvXs5MNKWf28U9gzSBlJkOyMl9LA1TZEjRtwvGXfP0Sl90g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.20.0", + "@smithy/middleware-endpoint": "^4.4.1", + "@smithy/middleware-stack": "^4.2.7", + "@smithy/protocol-http": "^5.3.7", + "@smithy/types": "^4.11.0", + "@smithy/util-stream": "^4.5.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.11.0.tgz", + "integrity": "sha512-mlrmL0DRDVe3mNrjTcVcZEgkFmufITfUAPBEA+AHYiIeYyJebso/He1qLbP3PssRe22KUzLRpQSdBPbXdgZ2VA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/url-parser": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.7.tgz", + "integrity": "sha512-/RLtVsRV4uY3qPWhBDsjwahAtt3x2IsMGnP5W1b2VZIe+qgCqkLxI1UOHDZp1Q1QSOrdOR32MF3Ph2JfWT1VHg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/querystring-parser": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-base64": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.3.0.tgz", + "integrity": "sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-body-length-browser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.0.tgz", + "integrity": "sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-body-length-node": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.1.tgz", + "integrity": "sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.2.0.tgz", + "integrity": "sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-config-provider": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.0.tgz", + "integrity": "sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-browser": { + "version": "4.3.16", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.16.tgz", + "integrity": "sha512-/eiSP3mzY3TsvUOYMeL4EqUX6fgUOj2eUOU4rMMgVbq67TiRLyxT7Xsjxq0bW3OwuzK009qOwF0L2OgJqperAQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/property-provider": "^4.2.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-node": { + "version": "4.2.19", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.19.tgz", + "integrity": "sha512-3a4+4mhf6VycEJyHIQLypRbiwG6aJvbQAeRAVXydMmfweEPnLLabRbdyo/Pjw8Rew9vjsh5WCdhmDaHkQnhhhA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/config-resolver": "^4.4.5", + "@smithy/credential-provider-imds": "^4.2.7", + "@smithy/node-config-provider": "^4.3.7", + "@smithy/property-provider": "^4.2.7", + "@smithy/smithy-client": "^4.10.2", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-endpoints": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.2.7.tgz", + "integrity": "sha512-s4ILhyAvVqhMDYREeTS68R43B1V5aenV5q/V1QpRQJkCXib5BPRo4s7uNdzGtIKxaPHCfU/8YkvPAEvTpxgspg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/node-config-provider": "^4.3.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-hex-encoding": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.0.tgz", + "integrity": "sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-middleware": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.7.tgz", + "integrity": "sha512-i1IkpbOae6NvIKsEeLLM9/2q4X+M90KV3oCFgWQI4q0Qz+yUZvsr+gZPdAEAtFhWQhAHpTsJO8DRJPuwVyln+w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-retry": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.7.tgz", + "integrity": "sha512-SvDdsQyF5CIASa4EYVT02LukPHVzAgUA4kMAuZ97QJc2BpAqZfA4PINB8/KOoCXEw9tsuv/jQjMeaHFvxdLNGg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/service-error-classification": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-stream": { + "version": "4.5.8", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.8.tgz", + "integrity": "sha512-ZnnBhTapjM0YPGUSmOs0Mcg/Gg87k503qG4zU2v/+Js2Gu+daKOJMeqcQns8ajepY8tgzzfYxl6kQyZKml6O2w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/fetch-http-handler": "^5.3.8", + "@smithy/node-http-handler": "^4.4.7", + "@smithy/types": "^4.11.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-hex-encoding": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-uri-escape": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.0.tgz", + "integrity": "sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.0.tgz", + "integrity": "sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-waiter": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.7.tgz", + "integrity": "sha512-vHJFXi9b7kUEpHWUCY3Twl+9NPOZvQ0SAi+Ewtn48mbiJk4JY9MZmKQjGB4SCvVb9WPiSphZJYY6RIbs+grrzw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/abort-controller": "^4.2.7", + "@smithy/types": "^4.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/uuid": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.0.tgz", + "integrity": "sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/bowser": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.13.1.tgz", + "integrity": "sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw==", + "license": "MIT" + }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + } + } +} diff --git a/scripts/storage/package.json b/scripts/storage/package.json new file mode 100644 index 00000000..1cfdaadb --- /dev/null +++ b/scripts/storage/package.json @@ -0,0 +1,12 @@ +{ + "name": "@fedeo/storage-migration", + "version": "1.0.0", + "private": true, + "type": "module", + "engines": { "node": ">=22" }, + "scripts": { "test": "node --test tests/*.test.mjs" }, + "dependencies": { + "@aws-sdk/client-s3": "3.962.0", + "@aws-sdk/s3-request-presigner": "3.962.0" + } +} diff --git a/scripts/storage/pilot.py b/scripts/storage/pilot.py new file mode 100644 index 00000000..70896972 --- /dev/null +++ b/scripts/storage/pilot.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Entry point shipped inside a verified, self-contained storage pilot bundle.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parent + + +def checksum(path): + h = hashlib.sha256() + with path.open("rb") as f: + for block in iter(lambda: f.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["load-images", "plan", "migrate", "activate", "resume-start", "rollback", "status"]) + parser.add_argument("--directory", default="/opt/fedeo") + parser.add_argument("--writers-exclusive", action="store_true") + args = parser.parse_args() + manifest = json.loads((ROOT / "bundle.json").read_text()) + for name, expected in manifest["sha256"].items(): + # The large image archive is checked when loading, not on every status call. + if name == "images.tar" and args.command != "load-images": + continue + path = ROOT / name + if path.resolve().is_relative_to(ROOT) and checksum(path) == expected: + continue + raise SystemExit("Paket-Prüfsumme stimmt nicht: " + name) + if args.command == "load-images": + subprocess.run(["docker", "image", "load", "-i", str(ROOT / "images.tar")], check=True) + for image in (manifest["toolsImage"], manifest["seaweedImage"]): + subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL) + print("Offline-Images geprüft und geladen. Nächster Schritt: plan.") + return + command = [sys.executable, str(ROOT / "scripts/selfhost-storage-migrate.py"), args.command, "--directory", args.directory] + if args.command == "plan": + command += ["--tools-image", manifest["toolsImage"]] + if args.writers_exclusive: + command += ["--writers-exclusive"] + raise SystemExit(subprocess.run(command).returncode) + + +if __name__ == "__main__": + main() diff --git a/scripts/storage/s3.mjs b/scripts/storage/s3.mjs new file mode 100644 index 00000000..9da4c33c --- /dev/null +++ b/scripts/storage/s3.mjs @@ -0,0 +1,283 @@ +import { createRequire } from 'node:module'; +import { createHash, randomUUID } from 'node:crypto'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { mkdir, open, readFile, rename, rm } from 'node:fs/promises'; +import { Transform } from 'node:stream'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { join } from 'node:path'; + +// A pilot also runs this contract inside the exact, already installed backend +// image. Its own SDK dependency tree must pass before the host can cut over. +const requireSdk = createRequire(process.env.FEDEO_STORAGE_SDK_ROOT + ? join(process.env.FEDEO_STORAGE_SDK_ROOT, 'package.json') : import.meta.url); +const S3 = requireSdk('@aws-sdk/client-s3'); +const { getSignedUrl } = requireSdk('@aws-sdk/s3-request-presigner'); + +// The pilot deliberately has a tested bound; no untested multipart conversion. +export const MAX_OBJECT_BYTES = 256 * 1024 * 1024; +export const MAX_OBJECTS = 200_000; +export const client = config => new S3.S3Client({ ...config, forcePathStyle: true, maxAttempts: 3 }); +export const call = (s3, operation, input) => s3.send(new S3[`${operation}Command`](input)); +export const canonical = value => JSON.stringify(value, (_, v) => v && typeof v === 'object' && !Array.isArray(v) + ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a.localeCompare(b))) : v); +export const digest = value => createHash('sha256').update(canonical(value)).digest('hex'); +const requireThat = (condition, reason) => { if (!condition) throw new Error(reason); }; + +export async function atomicJson(path, value) { + const temp = `${path}.tmp`; + const file = await open(temp, 'w', 0o600); + try { await file.writeFile(JSON.stringify(value, null, 2) + '\n'); await file.sync(); } + finally { await file.close(); } + await rename(temp, path); + const parent = await open(join(path, '..'), 'r'); + try { await parent.sync(); } finally { await parent.close(); } +} + +export function validateKeys(objects) { + requireThat(objects.length <= MAX_OBJECTS, `Pilotgrenze: höchstens ${MAX_OBJECTS} Objekte.`); + const keys = new Set(objects.map(o => o.Key)); + requireThat(keys.size === objects.length, 'Doppelte Objektschlüssel.'); + for (const { Key: key, Size: size } of objects) { + requireThat(typeof key === 'string' && key.length > 0 && Buffer.byteLength(key) <= 1024, + 'Ungültiger Objektschlüssel.'); + const segments = key.split('/'); + requireThat(!/[\x00-\x1f\x7f\\]/.test(key) && segments.every(s => s && s !== '.' && s !== '..'), + `Nicht verlustfrei übertragbarer Schlüssel: ${JSON.stringify(key)}`); + requireThat(Number.isSafeInteger(size) && size >= 0 && size <= MAX_OBJECT_BYTES, + `Objekt außerhalb der getesteten Größe (256 MiB): ${JSON.stringify(key)}`); + for (let i = 1; i < segments.length; i++) { + requireThat(!keys.has(segments.slice(0, i).join('/')), `Datei-/Verzeichniskonflikt: ${JSON.stringify(key)}`); + } + } +} + +export async function list(s3, Bucket) { + const result = []; + let ContinuationToken; + do { + const page = await call(s3, 'ListObjectsV2', { Bucket, ContinuationToken }); + result.push(...(page.Contents || []).map(o => ({ Key: o.Key, Size: o.Size, ETag: o.ETag, + LastModified: o.LastModified?.toISOString() }))); + requireThat(result.length <= MAX_OBJECTS, `Pilotgrenze: höchstens ${MAX_OBJECTS} Objekte.`); + requireThat(!page.IsTruncated || (page.NextContinuationToken && page.NextContinuationToken !== ContinuationToken), + 'Ungültige S3-Paginierung.'); + ContinuationToken = page.IsTruncated ? page.NextContinuationToken : undefined; + } while (ContinuationToken); + return result.sort((a, b) => Buffer.compare(Buffer.from(a.Key), Buffer.from(b.Key))); +} + +async function optional(s3, operation, Bucket, absentCodes) { + try { return await call(s3, operation, { Bucket }); } + catch (error) { if (absentCodes.includes(error.name)) return {}; throw error; } +} + +function privateAcl(acl) { + return (acl.Grants || []).every(g => g.Grantee?.Type === 'CanonicalUser' && + (g.Grantee.ID || '') === (acl.Owner?.ID || '') && g.Permission === 'FULL_CONTROL'); +} + +export async function inspectBucket(s3, Bucket) { + const buckets = await call(s3, 'ListBuckets', {}); + requireThat(buckets.Buckets?.length === 1 && buckets.Buckets[0].Name === Bucket, + 'Die Quelle muss genau den FEDEO-Bucket enthalten; weitere Buckets separat klären.'); + const v = await call(s3, 'GetBucketVersioning', { Bucket }); + requireThat(!v.Status, 'Aktivierte oder ausgesetzte Versionierung wird nicht unterstützt.'); + const versions = await call(s3, 'ListObjectVersions', { Bucket, MaxKeys: 1000 }); + // Even for an unversioned bucket MinIO returns current versions with VersionId=null. + requireThat(!(versions.DeleteMarkers?.length) && (versions.Versions || []).every(x => + x.IsLatest && x.VersionId === 'null'), 'Alte Versionen oder Delete-Marker vorhanden.'); + // A versioning flag above is mandatory; still inspect all version pages to detect old history. + let page = versions; + while (page.IsTruncated) { + const next = await call(s3, 'ListObjectVersions', { Bucket, KeyMarker: page.NextKeyMarker, + VersionIdMarker: page.NextVersionIdMarker }); + requireThat(!(next.DeleteMarkers?.length) && (next.Versions || []).every(x => x.IsLatest && x.VersionId === 'null'), + 'Alte Versionen oder Delete-Marker vorhanden.'); + requireThat(next.NextKeyMarker !== page.NextKeyMarker || !next.IsTruncated, 'Ungültige Versionspaginierung.'); + page = next; + } + const lock = await optional(s3, 'GetObjectLockConfiguration', Bucket, ['ObjectLockConfigurationNotFoundError', 'ObjectLockConfigurationNotFound']); + requireThat(!lock.ObjectLockConfiguration?.ObjectLockEnabled, 'Object Lock wird nicht unterstützt.'); + const encryption = await optional(s3, 'GetBucketEncryption', Bucket, ['ServerSideEncryptionConfigurationNotFoundError']); + requireThat(!encryption.ServerSideEncryptionConfiguration?.Rules?.length, 'Bucket-Verschlüsselung benötigt einen eigenen Adapter.'); + const policy = await optional(s3, 'GetBucketPolicy', Bucket, ['NoSuchBucketPolicy']); + requireThat(!policy.Policy, 'Eigene Bucket-Policy benötigt eine gesonderte Übernahme.'); + const lifecycle = await optional(s3, 'GetBucketLifecycleConfiguration', Bucket, ['NoSuchLifecycleConfiguration']); + requireThat(!lifecycle.Rules?.length, 'Lifecycle-Regeln vorhanden.'); + const replication = await optional(s3, 'GetBucketReplication', Bucket, ['ReplicationConfigurationNotFoundError']); + requireThat(!replication.ReplicationConfiguration, 'Bucket-Replikation vorhanden.'); + const notification = await call(s3, 'GetBucketNotificationConfiguration', { Bucket }); + requireThat(!['TopicConfigurations', 'QueueConfigurations', 'LambdaFunctionConfigurations'].some(k => notification[k]?.length) + && !notification.EventBridgeConfiguration, 'Bucket-Benachrichtigungen vorhanden.'); + const uploads = await call(s3, 'ListMultipartUploads', { Bucket, MaxUploads: 1 }); + requireThat(!uploads.Uploads?.length, 'Unvollständige Multipart-Uploads vorhanden.'); + const tags = await optional(s3, 'GetBucketTagging', Bucket, ['NoSuchTagSet', 'NoSuchTagSetError']); + requireThat(!tags.TagSet?.length, 'Bucket-Tags vorhanden.'); + const cors = await optional(s3, 'GetBucketCors', Bucket, ['NoSuchCORSConfiguration']); + requireThat(!cors.CORSRules?.length, 'Eigene CORS-Konfiguration vorhanden.'); + requireThat(privateAcl(await call(s3, 'GetBucketAcl', { Bucket })), 'Bucket-ACL ist nicht privat.'); +} + +export function metadata(head) { + requireThat(!head.ServerSideEncryption && !head.SSECustomerAlgorithm && !head.ObjectLockMode && + !head.ObjectLockLegalHoldStatus && !head.WebsiteRedirectLocation && (!head.StorageClass || head.StorageClass === 'STANDARD'), + 'Nicht unterstützte Objektattribute gefunden.'); + const result = {}; + for (const key of ['ContentType', 'ContentDisposition', 'ContentEncoding', 'ContentLanguage', 'CacheControl', 'Metadata']) { + if (head[key] !== undefined) result[key] = head[key]; + } + result.Metadata ||= {}; + if (head.Expires) result.Expires = head.Expires.toISOString(); + return result; +} + +export async function inventory(s3, Bucket) { + await inspectBucket(s3, Bucket); + const objects = await list(s3, Bucket); + validateKeys(objects); + for (const object of objects) { + const input = { Bucket, Key: object.Key }; + const head = await call(s3, 'HeadObject', input); + requireThat(head.ETag === object.ETag && head.ContentLength === object.Size, 'Quelle während Inventarisierung verändert.'); + object.metadata = metadata(head); + requireThat(!(await call(s3, 'GetObjectTagging', input)).TagSet?.length, `Objekt-Tags vorhanden: ${JSON.stringify(object.Key)}`); + requireThat(privateAcl(await call(s3, 'GetObjectAcl', input)), 'Objekt-ACL ist nicht privat.'); + } + return { schemaVersion: 1, bucket: Bucket, count: objects.length, + bytes: objects.reduce((n, o) => n + o.Size, 0), objects }; +} + +export async function readHash(s3, Bucket, object, spool) { + const response = await call(s3, 'GetObject', { Bucket, Key: object.Key, ...(object.ETag ? { IfMatch: object.ETag } : {}) }); + const hash = createHash('sha256'); + let bytes = 0; + const inspect = new Transform({ transform(chunk, _, next) { + bytes += chunk.length; + if (bytes > MAX_OBJECT_BYTES) return next(new Error('Objekt überschreitet Größenlimit.')); + hash.update(chunk); next(null, chunk); + }}); + if (spool) await pipeline(response.Body, inspect, createWriteStream(spool, { mode: 0o600 })); + else for await (const chunk of response.Body) { bytes += chunk.length; hash.update(chunk); + requireThat(bytes <= MAX_OBJECT_BYTES, 'Objekt überschreitet Größenlimit.'); } + requireThat(bytes === object.Size, 'Objektgröße während Lesen verändert.'); + return { sha256: hash.digest('hex'), size: bytes, metadata: metadata(response) }; +} + +export async function transfer(source, target, Bucket, directory, { verifyOnly = false } = {}) { + await mkdir(directory, { recursive: true, mode: 0o700 }); + const current = await inventory(source, Bucket); + const manifestPath = join(directory, 'inventory.json'); + let previous; + try { previous = JSON.parse(await readFile(manifestPath, 'utf8')); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + requireThat(!previous || canonical(previous) === canonical(current), 'Quellinventar seit Beginn verändert.'); + const targetObjects = await list(target, Bucket); + requireThat(previous || targetObjects.length === 0, 'Ziel ist bei erster Kopie nicht leer.'); + const sourceKeys = new Set(current.objects.map(o => o.Key)); + requireThat(targetObjects.every(o => sourceKeys.has(o.Key)), 'Unerklärte zusätzliche Zielobjekte vorhanden.'); + if (!previous) await atomicJson(manifestPath, current); + const existing = new Map(targetObjects.map(o => [o.Key, o])); + const journal = await open(join(directory, 'objects.jsonl'), 'a', 0o600); + const hashes = []; + try { + for (const object of current.objects) { + const spool = join(directory, 'object.spool'); + try { + const original = await readHash(source, Bucket, object, verifyOnly ? undefined : spool); + requireThat(canonical(original.metadata) === canonical(object.metadata), 'Quellmetadaten verändert.'); + let copied; + if (existing.has(object.Key) && existing.get(object.Key).Size === object.Size) { + copied = await readHash(target, Bucket, { Key: object.Key, Size: object.Size }); + } + if (canonical(copied) !== canonical(original)) { + requireThat(!verifyOnly, `Integritätsfehler: ${JSON.stringify(object.Key)}`); + const meta = { ...object.metadata }; + if (meta.Expires) meta.Expires = new Date(meta.Expires); + await call(target, 'PutObject', { Bucket, Key: object.Key, ...meta, + ContentLength: object.Size, ChecksumSHA256: Buffer.from(original.sha256, 'hex').toString('base64'), + Body: object.Size === 0 ? Buffer.alloc(0) : createReadStream(spool) }); + copied = await readHash(target, Bucket, { Key: object.Key, Size: object.Size }); + requireThat(canonical(copied) === canonical(original), `Integritätsfehler nach Kopie: ${JSON.stringify(object.Key)}`); + } + hashes.push({ key: object.Key, ...original }); + await journal.writeFile(JSON.stringify({ key: object.Key, ...original, verifiedAt: new Date().toISOString() }) + '\n'); + await journal.sync(); + process.stdout.write(JSON.stringify({ phase: verifyOnly ? 'verifying' : 'copying', completed: hashes.length, total: current.count }) + '\n'); + } finally { await rm(spool, { force: true }); } + } + } finally { await journal.close(); } + const after = await inventory(source, Bucket); + requireThat(canonical(after) === canonical(current), 'Quelle während Kopie/Prüfung verändert.'); + const result = await list(target, Bucket); + requireThat(canonical(result.map(({ Key, Size }) => ({ Key, Size }))) === + canonical(current.objects.map(({ Key, Size }) => ({ Key, Size }))), 'Schlüsselmenge oder Größen weichen ab.'); + const report = { schemaVersion: 1, verifiedAt: new Date().toISOString(), count: current.count, bytes: current.bytes, + inventorySha256: digest(current), objectsSha256: digest(hashes), objects: hashes }; + await atomicJson(join(directory, 'verified.json'), report); + return report; +} + +export async function verifyManifest(target, Bucket, report) { + const objects = await list(target, Bucket); + requireThat(canonical(objects.map(o => ({ key: o.Key, size: o.Size }))) === + canonical(report.objects.map(o => ({ key: o.key, size: o.size }))), 'Restore: Schlüsselmenge/Größen abweichend.'); + for (const object of report.objects) { + requireThat(canonical(await readHash(target, Bucket, { Key: object.key, Size: object.size })) === + canonical({ sha256: object.sha256, size: object.size, metadata: object.metadata }), 'Restore: Hash/Metadaten abweichend.'); + } + return { count: report.count, bytes: report.bytes, restored: true }; +} + +export async function cleanupProbe(s3, Bucket, probePath) { + let record; + try { record = JSON.parse(await readFile(probePath, 'utf8')); } + catch (error) { if (error.code === 'ENOENT') return; throw error; } + requireThat(/^fedeo-contract-[0-9a-f-]{36}\//.test(record.key), 'Ungültiges Probejournal.'); + await call(s3, 'DeleteObject', { Bucket, Key: record.key }); + await call(s3, 'DeleteObject', { Bucket, Key: record.key + '.stream' }); +} + +export async function contract(s3, config, Bucket, { foreignBucket, probePath } = {}) { + if (probePath) await cleanupProbe(s3, Bucket, probePath); + const Key = `fedeo-contract-${randomUUID()}/Mandant ä + % #/Prüfung.pdf`; + if (probePath) await atomicJson(probePath, { key: Key }); + const body = Buffer.from('FEDEO S3 Prüfung\n'); + try { + await call(s3, 'PutObject', { Bucket, Key, Body: body, ContentType: 'application/pdf', + ContentDisposition: 'inline; filename="test.pdf"', CacheControl: 'private, max-age=0', Metadata: { test: 'fedeo' } }); + const head = await call(s3, 'HeadObject', { Bucket, Key }); + requireThat(head.ContentLength === body.length && head.Metadata.test === 'fedeo', 'HEAD-Vertrag fehlgeschlagen.'); + const got = await call(s3, 'GetObject', { Bucket, Key }); + requireThat(Buffer.from(await got.Body.transformToByteArray()).equals(body), 'GET-Vertrag fehlgeschlagen.'); + // Application exports use streams. Exercise exactly the SDK environment setting + // applied by the pilot Compose file, including empty streams. + for (const bytes of [Buffer.alloc(0), Buffer.alloc(128 * 1024, 42)]) { + await call(s3, 'PutObject', { Bucket, Key: Key + '.stream', ContentLength: bytes.length, + Body: Readable.from([bytes]), ContentType: 'application/zip' }); + const streamHead = await call(s3, 'HeadObject', { Bucket, Key: Key + '.stream' }); + requireThat(!streamHead.ContentEncoding && streamHead.ContentLength === bytes.length, 'Streaming-Metadaten abweichend.'); + const downloaded = await call(s3, 'GetObject', { Bucket, Key: Key + '.stream' }); + requireThat(Buffer.from(await downloaded.Body.transformToByteArray()).equals(bytes), 'Streaming-Inhalt abweichend.'); + } + const url = await getSignedUrl(s3, new S3.GetObjectCommand({ Bucket, Key }), { expiresIn: 86400 }); + const signed = await fetch(url); + requireThat(signed.ok && Buffer.from(await signed.arrayBuffer()).equals(body), 'Presigned-URL fehlgeschlagen.'); + const anonymous = await fetch(url.split('?')[0]); + await anonymous.arrayBuffer(); + requireThat([401, 403].includes(anonymous.status), 'Anonymer Objektzugriff erlaubt!'); + const wrong = client({ ...config, credentials: { accessKeyId: 'fedeo-invalid', secretAccessKey: 'fedeo-invalid-secret' } }); + try { await denied(wrong, 'GetObject', { Bucket, Key }); } finally { wrong.destroy(); } + if (foreignBucket) await denied(s3, 'ListObjectsV2', { Bucket: foreignBucket }); + await call(s3, 'DeleteObject', { Bucket, Key }); + try { await call(s3, 'HeadObject', { Bucket, Key }); throw new Error('DELETE nicht wirksam.'); } + catch (error) { if (error.$metadata?.httpStatusCode !== 404) throw error; } + } finally { await call(s3, 'DeleteObject', { Bucket, Key }); await call(s3, 'DeleteObject', { Bucket, Key: Key + '.stream' }); } + return { operations: ['PUT', 'HEAD', 'GET', 'DELETE', 'SigV4', 'presigned-24h', 'anonymous-denied', 'wrong-key-denied'] }; +} + +async function denied(s3, operation, input) { + try { const value = await call(s3, operation, input); value.Body?.destroy(); } + catch (error) { if ([401, 403].includes(error.$metadata?.httpStatusCode)) return; throw error; } + throw new Error('Nicht autorisierter Zugriff wurde erlaubt.'); +} diff --git a/scripts/storage/seaweedfs.json b/scripts/storage/seaweedfs.json new file mode 100644 index 00000000..6de421a7 --- /dev/null +++ b/scripts/storage/seaweedfs.json @@ -0,0 +1,16 @@ +{ + "image": "chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882", + "entrypoint": ["/bin/sh", "/etc/fedeo/start-seaweedfs.sh"], + "command": [ + "server", "-dir=/data", "-master.dir=/data/master", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", + "-master.telemetry=false", "-master.volumeSizeLimitMB=1024", "-volume.max=0", "-volume.minFreeSpace=1GiB", + "-filer", "-s3", "-s3.ip.bind=0.0.0.0", "-s3.port=9000", "-s3.config=/etc/fedeo/s3.json", + "-s3.iam=false", "-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.autoCreateBucket=false", + "-s3.allowDeleteBucketNotEmpty=false" + ], + "stop_grace_period": "60s", + "healthcheck": { + "test": ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:9000/ 2>&1 | grep -q ' 403 '"], + "interval": "5s", "timeout": "3s", "retries": 30, "start_period": "10s" + } +} diff --git a/scripts/storage/start-seaweedfs.sh b/scripts/storage/start-seaweedfs.sh new file mode 100644 index 00000000..27e1ae45 --- /dev/null +++ b/scripts/storage/start-seaweedfs.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +# Never start an anonymously writable gateway when its configuration is absent, +# truncated, or replaced. The host generated and validated this exact file. +test -n "${FEDEO_S3_CONFIG_SHA256:-}" +test -s /etc/fedeo/s3.json +actual_sha256="$(sha256sum /etc/fedeo/s3.json)" +test "${actual_sha256%% *}" = "$FEDEO_S3_CONFIG_SHA256" +exec /usr/bin/weed "$@" diff --git a/scripts/storage/sync-sdk-lock.mjs b/scripts/storage/sync-sdk-lock.mjs new file mode 100644 index 00000000..752bd554 --- /dev/null +++ b/scripts/storage/sync-sdk-lock.mjs @@ -0,0 +1,31 @@ +// Preserve the backend's exact transitive SDK graph; top-level version equality alone is insufficient. +import { readFileSync, writeFileSync } from 'node:fs'; +const backend = JSON.parse(readFileSync(new URL('../../backend/package-lock.json', import.meta.url))); +const manifest = JSON.parse(readFileSync(new URL('./package.json', import.meta.url))); +const packages = { '': { name: manifest.name, version: manifest.version, dependencies: manifest.dependencies, engines: manifest.engines } }; +function resolve(from, dependency) { + const parts = from.split('/').filter(Boolean); + while (true) { + const candidate = [...parts, 'node_modules', dependency].join('/'); + if (backend.packages[candidate]) return candidate; + if (!parts.length) return null; + parts.pop(); + } +} +const queue = Object.keys(manifest.dependencies).map(name => resolve('', name)); +while (queue.length) { + const path = queue.pop(); + if (!path) throw new Error('Missing dependency in backend lockfile'); + if (packages[path]) continue; + const entry = { ...backend.packages[path] }; + delete entry.dev; + packages[path] = entry; + for (const name of Object.keys(entry.dependencies || {})) queue.push(resolve(path, name)); + for (const name of Object.keys(entry.optionalDependencies || {})) { + const resolved = resolve(path, name); + if (resolved) queue.push(resolved); + } +} +writeFileSync(new URL('./package-lock.json', import.meta.url), JSON.stringify({ name: manifest.name, version: manifest.version, + lockfileVersion: 3, requires: true, packages: Object.fromEntries(Object.entries(packages).sort(([a], [b]) => a.localeCompare(b))) }, null, 2) + '\n'); +console.log(`Copied ${Object.keys(packages).length - 1} locked backend dependencies.`); diff --git a/scripts/storage/tests/Dockerfile.minio b/scripts/storage/tests/Dockerfile.minio new file mode 100644 index 00000000..ae873587 --- /dev/null +++ b/scripts/storage/tests/Dockerfile.minio @@ -0,0 +1,6 @@ +# Test fixture only. The pilot bundle uses the instance's already installed MinIO. +FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 +RUN wget -q -O /usr/local/bin/minio https://github.com/minio/minio/releases/download/RELEASE.2025-09-07T16-13-09Z/minio.linux-amd64.RELEASE.2025-09-07T16-13-09Z \ + && echo '7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f /usr/local/bin/minio' | sha256sum -c - \ + && chmod 755 /usr/local/bin/minio +ENTRYPOINT ["minio"] diff --git a/scripts/storage/tests/host_integration.py b/scripts/storage/tests/host_integration.py new file mode 100644 index 00000000..93db013c --- /dev/null +++ b/scripts/storage/tests/host_integration.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Real Docker host-state test with synthetic services, never a real FEDEO instance.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import uuid +import sys + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location("storage_migration", ROOT / "selfhost-storage-migrate.py") +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + + +def docker(*args): + return subprocess.check_output(["docker", *map(str, args)], text=True, stderr=subprocess.PIPE).strip() + + +def fixture(): + root = Path(tempfile.mkdtemp(prefix="fedeo-host-migration-")) + project = "fedeo-pilot-" + uuid.uuid4().hex[:8] + (root / "minio").mkdir() + (root / "postgres").mkdir() + (root / ".env").write_text("# synthetic test only\n") + secret = "test-with-$-interpolation" + env = {"S3_ENDPOINT": "http://minio:9000", "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo", + "S3_ACCESS_KEY": "test-user", "S3_SECRET_KEY": secret, "FEDEO_VERSION": "3.3.0", + "DATABASE_URL": "postgres://fedeo:test-db@db:5432/fedeo", "FEDEO_FILE_BACKEND": "s3"} + idle = {"image": "node:22-alpine", "command": ["node", "-e", "setInterval(()=>{},1000)"], + "restart": "unless-stopped", "networks": ["internal"], + "healthcheck": {"test": ["CMD", "node", "-e", "process.exit(0)"], "interval": "1s"}} + import copy + services = {n: copy.deepcopy(idle) for n in ("backend", "frontend", "traefik", "node-exporter", "createbuckets")} + services["createbuckets"].update(command=["node", "-e", "process.exit(0)"], restart="no") + services["backend"]["environment"] = env + services["backend"]["image"] = "fedeo-storage-tools:pilot" + services["backend"]["entrypoint"] = ["node"] + services["backend"]["command"] = ["-e", "setInterval(()=>{},1000)"] + services["backend"]["depends_on"] = {"minio": {"condition": "service_healthy"}, "db": {"condition": "service_healthy"}, + "createbuckets": {"condition": "service_completed_successfully"}} + services["minio"] = {"image": "fedeo-minio-fixture:2025-09-07", "restart": "unless-stopped", + "command": ["server", "/data", "--console-address", ":9001"], "networks": ["internal"], + "volumes": [str(root / "minio") + ":/data"], + "environment": {"MINIO_ROOT_USER": "test-user", "MINIO_ROOT_PASSWORD": secret}, + "healthcheck": {"test": ["CMD", "node", "-e", "fetch('http://127.0.0.1:9000/minio/health/live').then(r=>process.exit(r.ok?0:1))"], "interval": "1s"}} + services["db"] = {"image": "postgres:16-alpine", "restart": "unless-stopped", "networks": ["internal"], + "volumes": [str(root / "postgres") + ":/var/lib/postgresql/data"], + "environment": {"POSTGRES_USER": "fedeo", "POSTGRES_PASSWORD": "test-db", "POSTGRES_DB": "fedeo"}, + "healthcheck": {"test": ["CMD", "pg_isready", "-U", "fedeo"], "interval": "1s"}} + config = {"name": project, "services": services, "networks": {"internal": {"name": project + "-internal"}}} + (root / "docker-compose.yml").write_text(json.dumps(m.literal(config))) + return root, project + + +def seed(root, project, hidden=False): + code = """import { client, call } from './s3.mjs'; +const s=client({endpoint:'http://minio:9000',region:'eu-central-1',credentials:{accessKeyId:'test-user',secretAccessKey:'test-with-$-interpolation'}}); +await call(s,'CreateBucket',{Bucket:'fedeo'}); +await call(s,'PutObject',{Bucket:'fedeo',Key:'tenant/filesbyid/1/ä + % #.pdf',Body:Buffer.from('fixture'),ContentType:'application/pdf'});""" + if hidden: + code += "for(const Key of ['hidden','hidden/child']) await call(s,'PutObject',{Bucket:'fedeo',Key,Body:'fixture'});" + code += "s.destroy();" + docker("run", "--rm", "--pull", "never", "--network", project + "-internal", "--entrypoint", "node", "fedeo-storage-tools:pilot", "--input-type=module", "-e", code) + + +def run(case): + root, project = fixture() + base = ["compose", "-p", project, "-f", str(root / "docker-compose.yml")] + try: + docker(*base, "up", "-d", "--pull", "never", "--wait", "--wait-timeout", "90") + seed(root, project, hidden=case == "hidden-prefix") + migration = m.Migration(root, "fedeo-storage-tools:pilot") + with migration.engine.lock(): + if case == "hidden-prefix": + try: + migration.plan() + raise AssertionError("hidden key conflict must block planning") + except m.Error as error: + assert "konflikt" in str(error) + assert not (root / ".updates/status.json").exists() + print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True) + return + migration.plan() + assert migration.state["count"] == 1 + migration.migrate(True) + assert migration.state["phase"] == "verified" + assert m.u.read_json(root / ".updates/status.json")["recoveryRequired"] + # Simulate restart by reconstructing the process and persisted state. + migration = m.Migration(root) + if case == "rollback": + migration.rollback() + assert migration.state["phase"] == "rolled_back" + assert not m.u.read_json(root / ".updates/status.json").get("recoveryRequired") + assert migration.inspect(migration.state["containers"]["minio"]["id"])["State"]["Running"] + else: + migration.activate() + assert migration.state["phase"] == "complete" and migration.state["writesReleased"] + assert migration.state["restoreVerifiedAt"] + old = migration.inspect(migration.state["containers"]["minio"]["id"]) + assert not old["State"]["Running"] and old["HostConfig"]["RestartPolicy"]["Name"] == "no" + assert migration.state["network"] not in old["NetworkSettings"]["Networks"] + try: + migration.rollback() + raise AssertionError("rollback after writes must fail") + except m.Error: + pass + print(json.dumps({"case": case, "result": "passed", "evidence": str(root)}), flush=True) + except BaseException: + print("Failure evidence: " + str(root), flush=True) + log = root / ".updates/storage-migration/commands.log" + if log.exists(): + print(log.read_text()[-8000:], flush=True) + ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project, + "--filter", "label=com.docker.compose.service=objectstore").split() + for cid in ids: + logs = subprocess.run(["docker", "logs", "--tail", "60", cid], capture_output=True, text=True) + print(logs.stdout + logs.stderr, flush=True) + raise + finally: + ids = docker("ps", "-aq", "--filter", "label=com.docker.compose.project=" + project).split() + if ids: + docker("rm", "-f", *ids) + remaining = docker("ps", "-aq", "--filter", "label=io.fedeo.storage-job=" + str(root / ".updates/storage-migration")).split() + if remaining: + docker("rm", "-f", *remaining) + docker("network", "rm", project + "-internal") + + +if __name__ == "__main__": + for case in (sys.argv[1:] or ["hidden-prefix", "rollback", "activate"]): + assert case in ("hidden-prefix", "rollback", "activate") + run(case) diff --git a/scripts/storage/tests/integration.mjs b/scripts/storage/tests/integration.mjs new file mode 100644 index 00000000..018f7d8c --- /dev/null +++ b/scripts/storage/tests/integration.mjs @@ -0,0 +1,112 @@ +// Explicit integration run; never discovers or connects to an existing FEDEO stack. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, mkdir, writeFile, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Readable } from 'node:stream'; +import { randomUUID, createHash } from 'node:crypto'; +import { client, call, contract, inventory, transfer, verifyManifest, MAX_OBJECT_BYTES } from '../s3.mjs'; + +const docker = (...args) => execFileSync('docker', args, { encoding: 'utf8', timeout: 120_000, stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const spec = JSON.parse(await readFile(new URL('../seaweedfs.json', import.meta.url), 'utf8')); +const root = await mkdtemp(join(tmpdir(), 'fedeo-storage-contract-')); +const prefix = `fedeo-contract-${randomUUID().slice(0, 8)}`; +const containers = []; +const Bucket = 'fedeo'; +const credentials = { accessKeyId: 'fedeo-test-user', secretAccessKey: 'fedeo-test-secret-only' }; +const adminCredentials = { accessKeyId: 'fedeo-test-admin', secretAccessKey: 'fedeo-test-admin-secret' }; +process.env.AWS_REQUEST_CHECKSUM_CALCULATION = 'WHEN_REQUIRED'; +const config = endpoint => ({ endpoint, region: 'eu-central-1', credentials }); +let source, target, admin; +function run(name, ...args) { containers.push(name); return docker('run', '-d', '--pull', 'never', '--name', name, ...args); } +function endpoint(name) { return 'http://' + docker('port', name, '9000/tcp'); } +async function ready(s3) { + for (let i = 0; i < 90; i++) { + try { await call(s3, 'ListBuckets', {}); return; } + catch { await new Promise(resolve => setTimeout(resolve, 1000)); } + } + throw new Error('S3 did not become ready'); +} +async function weed(name, data) { + await mkdir(join(data, 'master'), { recursive: true }); + run(name, '-p', '127.0.0.1::9000', '-v', `${data}:/data`, '-v', `${root}/config:/etc/fedeo:ro`, + '-e', 'FEDEO_S3_CONFIG_SHA256=' + createHash('sha256').update(await readFile(join(root, 'config/s3.json'))).digest('hex'), + '--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command); + return endpoint(name); +} +try { + await mkdir(join(root, 'config')); + await writeFile(join(root, 'config/start-seaweedfs.sh'), await readFile(new URL('../start-seaweedfs.sh', import.meta.url))); + assert.throws(() => docker('run', '--rm', '--pull', 'never', '-v', `${root}/config:/etc/fedeo:ro`, + '--entrypoint', spec.entrypoint[0], spec.image, ...spec.entrypoint.slice(1), ...spec.command)); + await writeFile(join(root, 'config/s3.json'), JSON.stringify({ identities: [ + { name: 'admin', credentials: [adminCredentials], actions: ['Admin', 'Read', 'Write', 'List', 'Tagging'] }, + { name: 'fedeo', credentials: [credentials], actions: [`Read:${Bucket}`, `Write:${Bucket}`, `List:${Bucket}`] } + ] })); + // Seaweed's credential JSON uses accessKey/secretKey rather than SDK names. + const auth = JSON.parse(await readFile(join(root, 'config/s3.json'), 'utf8')); + for (const identity of auth.identities) identity.credentials = identity.credentials.map(c => ({ accessKey: c.accessKeyId, secretKey: c.secretAccessKey })); + await writeFile(join(root, 'config/s3.json'), JSON.stringify(auth)); + run(`${prefix}-source`, '-p', '127.0.0.1::9000', '-e', `MINIO_ROOT_USER=${credentials.accessKeyId}`, + '-e', `MINIO_ROOT_PASSWORD=${credentials.secretAccessKey}`, process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', + 'server', '/data', '--console-address', ':9001'); + const sourceConfig = config(endpoint(`${prefix}-source`)); + source = client(sourceConfig); + await ready(source); + await call(source, 'CreateBucket', { Bucket }); + await contract(source, sourceConfig, Bucket); + const targetConfig = config(await weed(`${prefix}-target`, join(root, 'target'))); + target = client(targetConfig); + admin = client({ ...targetConfig, credentials: adminCredentials }); + await ready(admin); + await call(admin, 'CreateBucket', { Bucket }); + await call(admin, 'CreateBucket', { Bucket: 'foreign-private' }); + await contract(target, targetConfig, Bucket, { foreignBucket: 'foreign-private' }); + // Confirm unauthenticated management ports are not reachable through the container IP. + const ip = JSON.parse(docker('inspect', `${prefix}-target`))[0].NetworkSettings.Networks.bridge.IPAddress; + for (const port of [9333, 19333, 8888, 18888, 8080, 18080, 8181, 9101]) { + assert.throws(() => docker('exec', `${prefix}-source`, 'node', '-e', + `const s=require('node:net').connect(${port},${JSON.stringify(ip)});s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));s.setTimeout(1500,()=>process.exit(1))`)); + } + for (let i = 0; i < 1003; i++) await call(source, 'PutObject', { Bucket, Key: `tenant/filesbyid/${i}/ä + % #.txt`, + Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } }); + await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' }); + await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES, + Body: Readable.from((function* () { for (let i = 0; i < 256; i++) yield Buffer.alloc(1024 * 1024, i); })()), + ContentType: 'application/zip', ContentDisposition: 'attachment; filename="large.zip"', + CacheControl: 'private', ContentLanguage: 'de', Expires: new Date('2030-01-01T00:00:00Z') }); + const before = await inventory(source, Bucket); + assert.equal(before.count, 1005); + await call(source, 'PutObject', { Bucket, Key: 'unsupported-marker/', Body: '' }); + await assert.rejects(inventory(source, Bucket), /Schlüssel/); + await call(source, 'DeleteObject', { Bucket, Key: 'unsupported-marker/' }); + const start = Date.now(); + const report = await transfer(source, target, Bucket, join(root, 'work')); + await call(target, 'PutObject', { Bucket, Key: 'empty', Body: 'corrupt' }); + await assert.rejects(transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true }), /Integritätsfehler/); + await transfer(source, target, Bucket, join(root, 'work')); // Resume repairs and re-reads, never trusts journal alone. + await transfer(source, target, Bucket, join(root, 'work'), { verifyOnly: true }); + docker('stop', '-t', '60', `${prefix}-target`); + docker('run', '--rm', '--pull', 'never', '-v', `${root}:/test`, '--entrypoint', '/bin/sh', spec.image, + '-ec', 'tar -cpf /test/backup.tar -C /test/target .; mkdir /test/restored; tar -xpf /test/backup.tar -C /test/restored'); + const restoredConfig = config(await weed(`${prefix}-restored`, join(root, 'restored'))); + const restored = client(restoredConfig); + const restoredAdmin = client({ ...restoredConfig, credentials: adminCredentials }); + try { + await ready(restoredAdmin); + await verifyManifest(restored, Bucket, report); + await contract(restored, restoredConfig, Bucket, { foreignBucket: 'foreign-private' }); + } finally { restored.destroy(); restoredAdmin.destroy(); } + console.log(JSON.stringify({ result: 'passed', objects: report.count, bytes: report.bytes, + seconds: (Date.now() - start) / 1000, seaweed: spec.image, source: docker('image', 'inspect', + process.env.FEDEO_MINIO_TEST_IMAGE || 'fedeo-minio-fixture:2025-09-07', '--format', '{{.Id}}'), + evidence: root }, null, 2)); +} catch (error) { + for (const name of containers) { try { console.error(docker('logs', '--tail', '12', name)); } catch {} } + throw error; +} finally { + source?.destroy(); target?.destroy(); admin?.destroy(); + for (const name of containers.reverse()) { try { docker('rm', '-f', name); } catch {} } + // Preserve the isolated backup and report for inspection; no production paths are touched. +} diff --git a/scripts/storage/tests/keys.test.mjs b/scripts/storage/tests/keys.test.mjs new file mode 100644 index 00000000..0a9373bf --- /dev/null +++ b/scripts/storage/tests/keys.test.mjs @@ -0,0 +1,37 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { validateKeys, MAX_OBJECT_BYTES, metadata } from '../s3.mjs'; +import { cleanupProbe, atomicJson } from '../s3.mjs'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +test('accepts FEDEO keys, special characters, empty objects and size boundary', () => { + validateKeys([{ Key: 'tenant/filesbyid/123/ä + %#.pdf', Size: 0 }, { Key: 'export.zip', Size: MAX_OBJECT_BYTES }]); +}); +test('blocks lossy paths and file/directory collisions regardless of ordering', () => { + for (const Key of ['/a', 'a/', 'a//b', 'a/../b', './a', 'a\\b', 'a\nb']) + assert.throws(() => validateKeys([{ Key, Size: 1 }])); + assert.throws(() => validateKeys([{ Key: 'a/b', Size: 1 }, { Key: 'a', Size: 1 }])); + assert.throws(() => validateKeys([{ Key: 'a', Size: MAX_OBJECT_BYTES + 1 }])); +}); +test('rejects encryption and nonportable attributes rather than dropping them', () => { + for (const value of [{ ServerSideEncryption: 'AES256' }, { ObjectLockMode: 'GOVERNANCE' }, + { StorageClass: 'GLACIER' }, { WebsiteRedirectLocation: '/other' }]) assert.throws(() => metadata(value)); + assert.deepEqual(metadata({ ContentType: 'application/pdf', Metadata: { custom: 'value' } }), + { ContentType: 'application/pdf', Metadata: { custom: 'value' } }); +}); + +test('interrupted contract cleanup deletes only its two journalled probe objects', async () => { + const directory = await mkdtemp(join(tmpdir(), 'fedeo-probe-test-')); + try { + const path = join(directory, 'probe.json'); + const key = 'fedeo-contract-11111111-1111-1111-1111-111111111111/test'; + await atomicJson(path, { key }); + const inputs = []; + await cleanupProbe({ send: async command => { inputs.push(command.input); return {}; } }, 'bucket', path); + assert.deepEqual(inputs, [{ Bucket: 'bucket', Key: key }, { Bucket: 'bucket', Key: key + '.stream' }]); + await atomicJson(path, { key: 'real-business-file.pdf' }); + await assert.rejects(cleanupProbe({ send: async () => { throw new Error('must not send'); } }, 'bucket', path), /Probejournal/); + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/scripts/storage/tests/test_host.py b/scripts/storage/tests/test_host.py new file mode 100644 index 00000000..276d9e61 --- /dev/null +++ b/scripts/storage/tests/test_host.py @@ -0,0 +1,125 @@ +import copy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock + +spec = importlib.util.spec_from_file_location("storage_migration", Path(__file__).resolve().parents[2] / "selfhost-storage-migrate.py") +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + + +class HostSafetyTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + for name in ("minio", "postgres"): + (self.root / name).mkdir() + self.config = {"services": {n: {} for n in ("db", "minio", "createbuckets", "backend", "frontend", "traefik", "node-exporter")}} + for name, path, target in (("minio", "minio", "/data"), ("db", "postgres", "/var/lib/postgresql/data")): + self.config["services"][name]["volumes"] = [{"type": "bind", "source": str(self.root / path), "target": target}] + self.config["services"]["minio"].update(networks={"internal": {}}, environment={"MINIO_ROOT_USER": "user", "MINIO_ROOT_PASSWORD": "secret"}) + self.config["services"]["db"]["environment"] = {"POSTGRES_USER": "u", "POSTGRES_PASSWORD": "p", "POSTGRES_DB": "db"} + self.config["services"]["backend"]["environment"] = {"DATABASE_URL": "postgres://u:p@db:5432/db", "S3_ENDPOINT": "http://minio:9000", + "S3_REGION": "eu-central-1", "S3_BUCKET": "fedeo", "S3_ACCESS_KEY": "user", "S3_SECRET_KEY": "secret"} + + def test_rejects_external_or_unbacked_storage_and_secret_drift(self): + m.validate_config(self.config, self.root) + mutations = [lambda c: c["services"]["minio"].update(ports=["9000:9000"]), + lambda c: c["services"]["backend"]["environment"].update(S3_ENDPOINT="https://external.example"), + lambda c: c["services"]["backend"]["environment"].update(S3_SECRET_KEY="different"), + lambda c: c["services"]["minio"]["volumes"][0].update(source="/elsewhere"), + lambda c: c["services"].update(unknown={}), + lambda c: c["services"]["backend"]["environment"].update(DATABASE_URL="postgres://u:p@db:5432/other")] + for mutate in mutations: + config = copy.deepcopy(self.config) + mutate(config) + with self.assertRaises(m.Error): + m.validate_config(config, self.root) + + def test_rejects_symlink_source(self): + (self.root / "minio").rmdir() + (self.root / "minio").symlink_to(self.root / "postgres") + with self.assertRaises(m.Error): + m.validate_config(self.config, self.root) + + def test_cold_index_detects_objects_hidden_by_minio_listing(self): + source = self.root / "minio" + (source / ".minio.sys").mkdir() + (source / ".minio.sys/format.json").write_text(json.dumps({"format": "xl-single", "xl": {"version": "3"}})) + (source / "fedeo/a").mkdir(parents=True) + (source / "fedeo/a/xl.meta").write_bytes(b"opaque") + m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}]}, exact=True) + (source / "fedeo/a/b").mkdir() + (source / "fedeo/a/b/xl.meta").write_bytes(b"hidden-from-S3-LIST") + with self.assertRaisesRegex(m.Error, "Konflikt|konflikt"): + m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}]}, exact=True) + (source / "fedeo/a/b/xl.meta").unlink() + with self.assertRaisesRegex(m.Error, "weichen ab"): + m.validate_minio_listing(self.root, "fedeo", {"objects": [{"Key": "a"}, {"Key": "missing"}]}, exact=True) + + def test_update_and_migration_share_exclusive_lock(self): + a, b = m.Migration(self.root), m.Migration(self.root) + with a.engine.lock(): + with self.assertRaises(m.Error): + with b.engine.lock(): + self.fail("lock must be exclusive") + + def test_failed_freeze_keeps_persistent_recovery_guard(self): + migration = m.Migration(self.root) + migration.state = {"startedAt": "test", "writesReleased": False, "containers": {"backend": {"id": "backend"}}} + def fail(*args, **kwargs): + state = m.u.read_json(migration.engine.home / "status.json") + self.assertTrue(state["recoveryRequired"]) + self.assertEqual(m.u.read_json(migration.home / "state.json")["phase"], "freezing") + raise m.Error("power loss") + migration.docker = fail + with self.assertRaises(m.Error): + migration.freeze() + self.assertFalse(m.u.read_json(migration.home / "state.json")["writesReleased"]) + + def test_backup_corruption_prevents_progress(self): + migration = m.Migration(self.root) + backup = migration.home / "backup" + backup.mkdir() + (backup / "files.tar").write_bytes(b"valid") + m.u.write_json(backup / "complete.json", {"sha256": {"files.tar": m.sha_file(backup / "files.tar")}}) + migration.check_backup() + (backup / "files.tar").write_bytes(b"corrupt") + with self.assertRaises(m.Error): + migration.check_backup() + + def test_start_failure_is_already_past_rollback_boundary(self): + migration = m.Migration(self.root) + migration.state = {"startedAt": "test", "phase": "verified", "network": "internal", "writesReleased": False, + "containers": {"minio": {"id": "old"}}} + for method in ("assert_frozen", "verify_original", "check_backup", "cleanup_helper", "helper", "docker", "compose"): + setattr(migration, method, Mock()) + migration.inspect = lambda _: {"NetworkSettings": {"Networks": {}}} + migration.restore_test = Mock() + migration.runner = Mock() + (migration.home / "target.json").write_text('{}') + def fail_start(): + self.assertTrue(m.u.read_json(migration.home / "state.json")["writesReleased"]) + raise m.Error("backend did not start") + migration.start_application = fail_start + from unittest.mock import patch + (migration.home / "credentials.json").write_text('{"bucket":"fedeo"}') + with patch.object(m, "validate_minio_listing"): + (migration.home / "copy").mkdir() + (migration.home / "copy/inventory.json").write_text('{}') + with self.assertRaises(m.Error): + migration.activate() + with self.assertRaises(m.Error): + migration.rollback() + + def test_compose_roundtrip_preserves_dollar_signs(self): + value = {"environment": {"KEY": "literal-$-$$-${NOT_AN_ENV_VAR}"}, "command": ["echo $VALUE"]} + self.assertEqual(m.resolved(m.literal(value)), value) + + +if __name__ == "__main__": + unittest.main()