Compare commits
9 Commits
8473221c13
...
v3.1.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 7e9f4409f6 | |||
| 0ff74cdcdf | |||
| f51f2b13a1 | |||
| 2e63af1aa2 | |||
| 74c0cf40b3 | |||
| 5ebd2d508c | |||
| f6996c1d7b | |||
| 9128fbb165 | |||
| bb94e73229 |
@@ -17,7 +17,10 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
- run: npm ci --ignore-scripts --no-audit --no-fund
|
- run: npm ci --ignore-scripts --no-audit --no-fund
|
||||||
- name: Test release planning and recovery
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.12'
|
||||||
|
- name: Test releases and selfhost updates
|
||||||
run: npm test
|
run: npm test
|
||||||
- name: Check Conventional Commits
|
- name: Check Conventional Commits
|
||||||
run: npm run commits:check
|
run: npm run commits:check
|
||||||
@@ -30,8 +33,8 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
ref: ${{ github.sha }}
|
ref: ${{ github.sha }}
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
# Prefer repository secrets; retain the existing variable during migration.
|
# dev uses the normal read-only checkout; only main needs a release writer.
|
||||||
token: ${{ secrets.RELEASE_TOKEN || secrets.CI_TOKEN || vars.CI_TOKEN }}
|
token: ${{ github.ref == 'refs/heads/main' && (secrets.RELEASE_TOKEN || vars.RELEASE_TOKEN || vars.CI_TOKEN || secrets.CI_TOKEN) || github.token }}
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
@@ -41,13 +44,14 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
registry: git.federspiel.tech
|
registry: git.federspiel.tech
|
||||||
username: flfeders
|
username: flfeders
|
||||||
password: ${{ secrets.CI_TOKEN || vars.CI_TOKEN }}
|
# Same credential source as the existing image-build workflows.
|
||||||
|
password: ${{ vars.CI_TOKEN || secrets.CI_TOKEN }}
|
||||||
- name: Build all images and publish main releases
|
- name: Build all images and publish main releases
|
||||||
# Gitea 1.25 ignores concurrency. ci.mjs uses an atomic Git ref lock instead.
|
# main uses a Git ref lock. dev only reads Git and writes to the registry.
|
||||||
env:
|
env:
|
||||||
CI: 'true'
|
CI: 'true'
|
||||||
FEDEO_REF: ${{ github.ref }}
|
FEDEO_REF: ${{ github.ref }}
|
||||||
FEDEO_SHA: ${{ github.sha }}
|
FEDEO_SHA: ${{ github.sha }}
|
||||||
FEDEO_EVENT_NAME: ${{ github.event_name }}
|
FEDEO_EVENT_NAME: ${{ github.event_name }}
|
||||||
GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN || secrets.CI_TOKEN || vars.CI_TOKEN }}
|
GITEA_TOKEN: ${{ github.ref == 'refs/heads/main' && (secrets.RELEASE_TOKEN || vars.RELEASE_TOKEN || vars.CI_TOKEN || secrets.CI_TOKEN) || '' }}
|
||||||
run: npm run release:ci
|
run: npm run release:ci
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -3,6 +3,7 @@ node_modules/
|
|||||||
.nuxt/
|
.nuxt/
|
||||||
.output/
|
.output/
|
||||||
/.release/
|
/.release/
|
||||||
|
__pycache__/
|
||||||
|
|
||||||
# Lokale Runtime-Daten und generierte Konfigurationen
|
# Lokale Runtime-Daten und generierte Konfigurationen
|
||||||
matrix/postgres/
|
matrix/postgres/
|
||||||
|
|||||||
40
README.md
40
README.md
@@ -487,49 +487,25 @@ Wenn der Bootstrap aktiviert ist, kannst du dich danach mit `FEDEO_BOOTSTRAP_ADM
|
|||||||
|
|
||||||
## Updates
|
## Updates
|
||||||
|
|
||||||
Script-installierte Instanzen liegen standardmassig unter `/opt/fedeo`. Ein Update besteht daraus, die Selfhost-Dateien zu erneuern, die vorgebauten Images zu ziehen und den Compose-Stack neu zu starten.
|
Vollständige Release-Updates einschließlich Compose-Datei werden mit dem Host-Skript vorbereitet und ausgeführt:
|
||||||
|
|
||||||
Vor kritischen Updates sollte ein Backup der persistenten Daten erstellt werden:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /opt/fedeo
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py check
|
||||||
tar -czf "backup-$(date +%Y%m%d-%H%M%S).tar.gz" .env postgres minio
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py apply --version 3.1.0
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py status
|
||||||
```
|
```
|
||||||
|
|
||||||
Danach die aktuellen Selfhost-Dateien laden und den Stack aktualisieren:
|
`3.1.0` durch den gewünschten veröffentlichten Release ersetzen. Das Skript benötigt Python ab 3.10 und Docker Compose ab 2.24. Es sichert die lokale Datenbank, MinIO, Konfiguration und Zertifikate vor dem Austausch, erhält `.env` und prüft die gestarteten Versionen. Während Backup und Aktualisierung gibt es eine Wartungsunterbrechung.
|
||||||
|
|
||||||
```bash
|
Beim ersten Update die erzeugte Compose-Differenz prüfen, eigene Anpassungen in eine Override-Datei übertragen und den Austausch mit `--accept-compose-replacement` bestätigen. Bestehende Installationen müssen das Update-Skript zunächst aus einem passenden Release übernehmen.
|
||||||
cd /opt/fedeo
|
|
||||||
curl -fsSL https://git.federspiel.tech/flfeders/FEDEO/raw/branch/dev/docker-compose.selfhost.yml -o /opt/fedeo/docker-compose.yml
|
|
||||||
curl -fsSL https://git.federspiel.tech/flfeders/FEDEO/raw/branch/dev/.env.example -o /opt/fedeo/.env.example
|
|
||||||
curl -fsSL https://git.federspiel.tech/flfeders/FEDEO/raw/branch/dev/scripts/selfhost-setup.sh -o /opt/fedeo/scripts/selfhost-setup.sh
|
|
||||||
chmod +x /opt/fedeo/scripts/selfhost-setup.sh
|
|
||||||
docker compose --env-file /opt/fedeo/.env -f /opt/fedeo/docker-compose.yml pull
|
|
||||||
docker compose --env-file /opt/fedeo/.env -f /opt/fedeo/docker-compose.yml up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
Der Backend-Container wendet Datenbankmigrationen beim Start automatisch an, solange `FEDEO_RUN_MIGRATIONS` nicht deaktiviert wurde. Falls Migrationen manuell erzwungen werden sollen:
|
Voraussetzungen, unterstützte Konfigurationen und Wiederherstellung stehen in [Vollständige Selfhost-Updates](docs/selfhost-updates.md). Für ältere Releases steht der manuelle Ablauf in [Versionierung und Releases](docs/releases.md).
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose --env-file /opt/fedeo/.env -f /opt/fedeo/docker-compose.yml run --rm backend npm run migrate
|
|
||||||
docker compose --env-file /opt/fedeo/.env -f /opt/fedeo/docker-compose.yml up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
Anschliessend den Zustand prüfen:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose --env-file /opt/fedeo/.env -f /opt/fedeo/docker-compose.yml ps
|
|
||||||
curl -I https://app.example.com
|
|
||||||
curl https://app.example.com/backend/health
|
|
||||||
```
|
|
||||||
|
|
||||||
Die Selfhost-Compose-Datei nutzt vorgebaute Images. Dadurch braucht der Server keinen Repository-Checkout und keine lokalen Build-Kontexte.
|
|
||||||
|
|
||||||
## Backup-Empfehlung
|
## Backup-Empfehlung
|
||||||
|
|
||||||
Regelmassig sichern:
|
Regelmassig sichern:
|
||||||
|
|
||||||
- `./postgres`
|
- PostgreSQL mit einem konsistenten Datenbankbackup, beispielsweise `pg_dump`
|
||||||
- `./minio` falls MinIO lokal genutzt wird
|
- `./minio` falls MinIO lokal genutzt wird
|
||||||
- `./traefik/letsencrypt/acme.json`
|
- `./traefik/letsencrypt/acme.json`
|
||||||
- deine `.env`
|
- deine `.env`
|
||||||
|
|||||||
@@ -98,10 +98,6 @@ async function main() {
|
|||||||
done()
|
done()
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get('/health', async (req, res) => {
|
|
||||||
return res.send({ status: 'ok' })
|
|
||||||
})
|
|
||||||
|
|
||||||
//Plugin nur auf bestimmten Routes
|
//Plugin nur auf bestimmten Routes
|
||||||
await app.register(queryConfigPlugin, {
|
await app.register(queryConfigPlugin, {
|
||||||
routes: ['/api/resource/:resource/paginated']
|
routes: ['/api/resource/:resource/paginated']
|
||||||
|
|||||||
@@ -2,6 +2,16 @@ import { FastifyInstance } from "fastify";
|
|||||||
import { getBuildInfo } from "../utils/buildInfo";
|
import { getBuildInfo } from "../utils/buildInfo";
|
||||||
|
|
||||||
export default async function routes(server: FastifyInstance) {
|
export default async function routes(server: FastifyInstance) {
|
||||||
|
server.get("/health", async (_request, reply) => {
|
||||||
|
reply.header("Cache-Control", "no-store");
|
||||||
|
try {
|
||||||
|
await server.db.execute("SELECT 1");
|
||||||
|
return { status: "ok", ...getBuildInfo() };
|
||||||
|
} catch {
|
||||||
|
return reply.code(503).send({ status: "unavailable" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
server.get("/version", async (_request, reply) => {
|
server.get("/version", async (_request, reply) => {
|
||||||
reply.header("Cache-Control", "no-store");
|
reply.header("Cache-Control", "no-store");
|
||||||
return getBuildInfo();
|
return getBuildInfo();
|
||||||
|
|||||||
@@ -3,6 +3,28 @@ import { test } from 'node:test';
|
|||||||
import Fastify from 'fastify';
|
import Fastify from 'fastify';
|
||||||
import healthRoutes from '../src/routes/health';
|
import healthRoutes from '../src/routes/health';
|
||||||
|
|
||||||
|
test('health checks the database and reports a failure without leaking connection details', async () => {
|
||||||
|
const app = Fastify();
|
||||||
|
let available = true;
|
||||||
|
app.decorate('db', { execute: async () => {
|
||||||
|
if (!available) throw new Error('postgres://secret@database');
|
||||||
|
return { rows: [{ result: 1 }] };
|
||||||
|
} } as any);
|
||||||
|
try {
|
||||||
|
await app.register(healthRoutes);
|
||||||
|
const healthy = await app.inject('/health');
|
||||||
|
assert.equal(healthy.statusCode, 200);
|
||||||
|
assert.equal(healthy.json().status, 'ok');
|
||||||
|
assert.equal(healthy.headers['cache-control'], 'no-store');
|
||||||
|
available = false;
|
||||||
|
const unhealthy = await app.inject('/health');
|
||||||
|
assert.equal(unhealthy.statusCode, 503);
|
||||||
|
assert.deepEqual(unhealthy.json(), { status: 'unavailable' });
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('version endpoint reports image metadata without a database or authentication', async () => {
|
test('version endpoint reports image metadata without a database or authentication', async () => {
|
||||||
const previousVersion = process.env.FEDEO_VERSION;
|
const previousVersion = process.env.FEDEO_VERSION;
|
||||||
const previousRevision = process.env.FEDEO_REVISION;
|
const previousRevision = process.env.FEDEO_REVISION;
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ services:
|
|||||||
- internal
|
- internal
|
||||||
|
|
||||||
minio:
|
minio:
|
||||||
image: minio/minio:latest
|
image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||||
container_name: fedeo-minio
|
container_name: fedeo-minio
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: server /data --console-address ":9001"
|
command: server /data --console-address ":9001"
|
||||||
@@ -66,13 +66,13 @@ services:
|
|||||||
- internal
|
- internal
|
||||||
|
|
||||||
createbuckets:
|
createbuckets:
|
||||||
image: minio/mc:latest
|
image: quay.io/minio/mc:RELEASE.2025-08-13T08-35-41Z@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727
|
||||||
container_name: fedeo-minio-init
|
container_name: fedeo-minio-init
|
||||||
depends_on:
|
depends_on:
|
||||||
minio:
|
minio:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
entrypoint: >
|
entrypoint: >
|
||||||
/bin/sh -c "
|
/bin/sh -ec "
|
||||||
mc alias set local http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD};
|
mc alias set local http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD};
|
||||||
mc mb --ignore-existing local/${MINIO_BUCKET};
|
mc mb --ignore-existing local/${MINIO_BUCKET};
|
||||||
mc anonymous set private local/${MINIO_BUCKET};
|
mc anonymous set private local/${MINIO_BUCKET};
|
||||||
@@ -143,6 +143,12 @@ services:
|
|||||||
FEDEO_BOOTSTRAP_TENANT_NAME: ${FEDEO_BOOTSTRAP_TENANT_NAME:-FEDEO}
|
FEDEO_BOOTSTRAP_TENANT_NAME: ${FEDEO_BOOTSTRAP_TENANT_NAME:-FEDEO}
|
||||||
FEDEO_BOOTSTRAP_TENANT_SHORT: ${FEDEO_BOOTSTRAP_TENANT_SHORT:-FEDEO}
|
FEDEO_BOOTSTRAP_TENANT_SHORT: ${FEDEO_BOOTSTRAP_TENANT_SHORT:-FEDEO}
|
||||||
NODE_EXPORTER_URL: ${NODE_EXPORTER_URL:-http://node-exporter:9100}
|
NODE_EXPORTER_URL: ${NODE_EXPORTER_URL:-http://node-exporter:9100}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:' + (process.env.PORT || '3100') + '/health', {signal: AbortSignal.timeout(4000)}).then(r => {if (!r.ok) process.exit(1)}).catch(() => process.exit(1))"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 6
|
||||||
|
start_period: 120s
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
- traefik.http.routers.fedeo-backend.rule=Host(`${DOMAIN}`) && PathPrefix(`/backend`)
|
- traefik.http.routers.fedeo-backend.rule=Host(`${DOMAIN}`) && PathPrefix(`/backend`)
|
||||||
@@ -178,11 +184,18 @@ services:
|
|||||||
container_name: fedeo-frontend
|
container_name: fedeo-frontend
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
depends_on:
|
depends_on:
|
||||||
- backend
|
backend:
|
||||||
|
condition: service_healthy
|
||||||
environment:
|
environment:
|
||||||
NODE_ENV: production
|
NODE_ENV: production
|
||||||
NUXT_PUBLIC_API_BASE: https://${DOMAIN}/backend
|
NUXT_PUBLIC_API_BASE: https://${DOMAIN}/backend
|
||||||
NUXT_PUBLIC_PDF_LICENSE: ${NUXT_PUBLIC_PDF_LICENSE}
|
NUXT_PUBLIC_PDF_LICENSE: ${NUXT_PUBLIC_PDF_LICENSE}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/api/health', {signal: AbortSignal.timeout(4000)}).then(r => {if (!r.ok) process.exit(1)}).catch(() => process.exit(1))"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 6
|
||||||
|
start_period: 30s
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
- traefik.http.routers.fedeo-frontend.rule=Host(`${DOMAIN}`)
|
- traefik.http.routers.fedeo-frontend.rule=Host(`${DOMAIN}`)
|
||||||
|
|||||||
@@ -41,7 +41,9 @@ Die Pipeline nutzt die fest gepinnten Pakete `@semantic-release/commit-analyzer`
|
|||||||
|
|
||||||
Es zählt der höchste erforderliche Sprung seit dem letzten stabilen Tag `vMAJOR.MINOR.PATCH`. Eine Änderung und ihr vollständiger Git-Revert innerhalb desselben Release-Bereichs heben sich auf. Selbst verfasste Nachrichten müssen dem Schema entsprechen; echte Merge-Commits und automatisch generierte Git-Reverts werden gesondert behandelt.
|
Es zählt der höchste erforderliche Sprung seit dem letzten stabilen Tag `vMAJOR.MINOR.PATCH`. Eine Änderung und ihr vollständiger Git-Revert innerhalb desselben Release-Bereichs heben sich auf. Selbst verfasste Nachrichten müssen dem Schema entsprechen; echte Merge-Commits und automatisch generierte Git-Reverts werden gesondert behandelt.
|
||||||
|
|
||||||
Der erste Release heißt `3.0.0`. `bootstrapCommit` in `release.config.json` begrenzt die Prüfung und die erste Änderungsliste auf die Einführung dieser Versionierung. Ältere Commits werden weder umgeschrieben noch nachträglich klassifiziert. Den Ausgangscommit nach Einführung nicht ändern.
|
Der erste Release heißt `3.0.0`. `bootstrapCommit` in `release.config.json` begrenzt die Prüfung und die erste Änderungsliste auf die Einführung dieser Versionierung auf `dev`. `legacyHistoryHeads` hält zusätzlich den letzten alten `main`-Stand fest (`e7554fa2…` vor der ersten Übernahme von `dev`). Dessen Historie enthält ältere Änderungen an Issue-Vorlagen, die nicht vom `dev`-Ausgangscommit erreichbar sind. Beide festgelegten Historien werden aus Commit-Prüfung, Versionsberechnung und Release Notes ausgeschlossen. Neue Commits zählen auch dann, wenn ihr Zweig von einem dieser alten Stände abzweigt.
|
||||||
|
|
||||||
|
Ältere Commits werden weder umgeschrieben noch nachträglich klassifiziert. Diese festen Ausgangscommits nach Einführung nicht verschieben und keine späteren Änderungen hinzufügen, um die Commit-Prüfung zu umgehen. Der vollständige Checkout muss auch die alten Branch-Historien enthalten; ein fehlender Ausgangscommit führt zum Abbruch.
|
||||||
|
|
||||||
## Versionen und Artefakte
|
## Versionen und Artefakte
|
||||||
|
|
||||||
@@ -49,9 +51,9 @@ Backend, Frontend, Dokumentation, Website und zentrale Dienste werden als gemein
|
|||||||
|
|
||||||
- Stabile Images: `…/backend:3.2.0`, `…/backend:v3.2.0` und `…/backend:sha-<vollständiger Commit>`; entsprechend für alle sechs Komponenten.
|
- Stabile Images: `…/backend:3.2.0`, `…/backend:v3.2.0` und `…/backend:sha-<vollständiger Commit>`; entsprechend für alle sechs Komponenten.
|
||||||
- Bewegliche Kanäle: `latest` und `main` zeigen nach erfolgreicher Veröffentlichung auf denselben Release.
|
- Bewegliche Kanäle: `latest` und `main` zeigen nach erfolgreicher Veröffentlichung auf denselben Release.
|
||||||
- Entwicklung: `dev` sowie unveränderliche `dev-sha-<vollständiger Commit>`-Images. Die eingebettete Entwicklungsversion enthält Commit-Anzahl und Commit-ID.
|
- Entwicklung: `dev` sowie unveränderliche `dev-sha-<vollständiger Commit>-<UUID>`-Images. Alle Komponenten eines erfolgreichen Laufs erhalten denselben Snapshot-Tag. Auch eine Wiederholung desselben Commits baut einen eigenen Snapshot. Die eingebettete Entwicklungsversion enthält Commit-Anzahl und Commit-ID.
|
||||||
- Jeder Build erhält zunächst einen eindeutigen temporären `build-<UUID>`-Tag. Diese Tags können später nach einer Aufbewahrungsfrist bereinigt werden; stabile Versionen und deren Digests müssen erhalten bleiben.
|
- Release-Builds erhalten zunächst einen eindeutigen temporären `build-<UUID>`-Tag. Diese Tags können später nach einer Aufbewahrungsfrist bereinigt werden; stabile Versionen und deren Digests müssen erhalten bleiben.
|
||||||
- Release-Dateien: `CHANGELOG.md`, `release-manifest.json`, `docker-compose.selfhost.yml`, `.env.example`, `selfhost-install.sh`, `selfhost-setup.sh` und `SHA256SUMS`.
|
- Release-Dateien: `CHANGELOG.md`, `release-manifest.json`, `docker-compose.selfhost.yml`, `.env.example`, `selfhost-install.sh`, `selfhost-setup.sh`, `selfhost-update.py`, `SELFHOST-UPDATES.md` und `SHA256SUMS`.
|
||||||
|
|
||||||
Das Manifest dokumentiert die Image-Digests, den Commit und SHA-256-Prüfsummen aller enthaltenen SQL-Migrationen. Es beschreibt den ausgelieferten Migrationsstand; den tatsächlich angewendeten Stand einer Installation bestimmt deren Datenbank.
|
Das Manifest dokumentiert die Image-Digests, den Commit und SHA-256-Prüfsummen aller enthaltenen SQL-Migrationen. Es beschreibt den ausgelieferten Migrationsstand; den tatsächlich angewendeten Stand einer Installation bestimmt deren Datenbank.
|
||||||
|
|
||||||
@@ -71,18 +73,22 @@ Beispiel: `{"product":"FEDEO","version":"3.2.0","revision":"<Commit-ID>"}`. Die
|
|||||||
|
|
||||||
Der vorhandene Actions-Runner benötigt Docker mit Buildx und Netzwerkzugriff auf npm, Basis-Images und die Registry. Alle Images werden wie der bisherige Backend-Build für `linux/amd64` erzeugt. Eine zusätzliche ARM-Variante ist ein eigener Ausbau.
|
Der vorhandene Actions-Runner benötigt Docker mit Buildx und Netzwerkzugriff auf npm, Basis-Images und die Registry. Alle Images werden wie der bisherige Backend-Build für `linux/amd64` erzeugt. Eine zusätzliche ARM-Variante ist ein eigener Ausbau.
|
||||||
|
|
||||||
Unter Repository → Settings → Actions → Secrets:
|
Unter Repository → Settings → Actions → Variables bzw. Secrets:
|
||||||
|
|
||||||
- `CI_TOKEN`: Token des Registry-Benutzers `flfeders`, mindestens mit `write:package`. Die bisherige Variable `vars.CI_TOKEN` wird als Übergang weiterhin unterstützt; langfristig als Secret hinterlegen.
|
- `CI_TOKEN`: Token des Registry-Benutzers `flfeders`, mindestens mit `write:package`. Die Registry-Anmeldung verwendet wie die bisherigen Build-Workflows vorrangig `vars.CI_TOKEN`; `secrets.CI_TOKEN` dient als Rückfallwert. Für eine spätere Umstellung auf ein Secret die gleichnamige Variable entfernen.
|
||||||
- Optional `RELEASE_TOKEN`: Token mit `write:repository` für Release-API und Git-Tags sowie zum Erzeugen/Löschen der temporären Sperr-Branches. Ohne separates Token wird `CI_TOKEN` dafür verwendet und benötigt dann zusätzlich diese Rechte.
|
- Optional `RELEASE_TOKEN` als Secret oder Variable: Token mit `write:repository` für Release-API und Git-Tags sowie zum Erzeugen/Löschen des temporären Sperr-Branches `automation/release-lock`. Nur `main` benötigt diese Rechte. Die Reihenfolge für Git-Anmeldung und Release-API ist `secrets.RELEASE_TOKEN`, `vars.RELEASE_TOKEN`, `vars.CI_TOKEN`, `secrets.CI_TOKEN`. Ohne separates Release-Token benötigt der verwendete `CI_TOKEN` zusätzlich diese Rechte.
|
||||||
|
|
||||||
Der Token-Benutzer muss die entsprechenden Repository-Rechte haben. Ein Gitea-Standard-Job-Token reicht je nach Server-Konfiguration nicht zum Pushen und zum Registry-Upload. Geschützte Tags `v*` müssen den Release-Benutzer zulassen; die Branches `automation/release-lock` und `automation/dev-build-lock` müssen durch ihn erzeugt und entfernt werden können. Die Pipeline benötigt keine Schreibrechte auf den Inhalt von `main`.
|
Auf `dev` nutzt der Checkout wie die bisherigen Workflows den normalen Job-Token zum Lesen. Entwicklungsbuilds pushen keine Git-Refs und rufen die Release-API nicht auf; der Registry-Token benötigt dafür keine Repository-Schreibrechte.
|
||||||
|
|
||||||
|
Der Release-Token-Benutzer muss die entsprechenden Repository-Rechte haben. Geschützte Tags `v*` müssen ihn zulassen; `automation/release-lock` muss durch ihn erzeugt und entfernt werden können. Die Pipeline benötigt keine Schreibrechte auf den Inhalt von `main`. Ein 403 beim Anlegen der Main-Sperre führt vor dem Image-Build zum Abbruch mit einem Hinweis auf die fehlenden Rechte.
|
||||||
|
|
||||||
`dev` mit der neuen Konfiguration zu pushen prüft zunächst den Entwicklungsbuild. Erst die bewusste Übernahme nach `main` erstellt den ersten stabilen Release. Release-Erstellung und Deployment haben eigene Auslöser: Server werden durch diese Pipeline nicht neu gestartet.
|
`dev` mit der neuen Konfiguration zu pushen prüft zunächst den Entwicklungsbuild. Erst die bewusste Übernahme nach `main` erstellt den ersten stabilen Release. Release-Erstellung und Deployment haben eigene Auslöser: Server werden durch diese Pipeline nicht neu gestartet.
|
||||||
|
|
||||||
## Veröffentlichungsreihenfolge und Wiederaufnahme
|
## Veröffentlichungsreihenfolge und Wiederaufnahme
|
||||||
|
|
||||||
Gitea 1.25 ignoriert das Workflow-Feld `concurrency`. Deshalb reserviert der Runner atomar einen Git-Branch per leerem `--force-with-lease`. Ein zweiter Runner wartet bis zu zwei Stunden, ohne die Sperre zu übernehmen. Für Entwicklungsbuilds gibt es eine separate Sperre. Veraltete Main-Läufe werden vor dem Build übersprungen, wenn bereits ein neuerer Main-Commit vorhanden ist; dessen Lauf enthält alle ausstehenden Änderungen.
|
Gitea 1.25 ignoriert das Workflow-Feld `concurrency`. Deshalb reserviert der Runner für Releases auf `main` atomar einen Git-Branch per leerem `--force-with-lease`. Ein zweiter Release-Runner wartet bis zu zwei Stunden, ohne die Sperre zu übernehmen. Veraltete Main-Läufe werden vor dem Build übersprungen, wenn bereits ein neuerer Main-Commit vorhanden ist; dessen Lauf enthält alle ausstehenden Änderungen.
|
||||||
|
|
||||||
|
Entwicklungsbuilds benötigen keine Git-Sperre. Parallele Läufe bauen unabhängige Snapshot-Tags. Nach dem vollständigen Build wird vor jeder Aktualisierung eines `dev`-Images geprüft, ob der Commit noch der aktuelle Stand von `origin/dev` ist. Ein erkennbar veralteter Lauf beendet die Aktualisierung. Diese Prüfung und die Registry-Aktualisierung sind nicht atomar; für einen festen Stand aller Komponenten denselben vollständigen Snapshot-Tag oder einen stabilen Release verwenden.
|
||||||
|
|
||||||
1. Vollständigen Commit und Tags laden, Commit-Nachrichten prüfen, Version berechnen.
|
1. Vollständigen Commit und Tags laden, Commit-Nachrichten prüfen, Version berechnen.
|
||||||
2. Alle sechs Images aus diesem Checkout mit Lockfiles bauen. Der Backend-Build führt zusätzlich seine Tests aus. Vorher laufen die Release-Tests in einem eigenen Job.
|
2. Alle sechs Images aus diesem Checkout mit Lockfiles bauen. Der Backend-Build führt zusätzlich seine Tests aus. Vorher laufen die Release-Tests in einem eigenen Job.
|
||||||
@@ -91,7 +97,7 @@ Gitea 1.25 ignoriert das Workflow-Feld `concurrency`. Deshalb reserviert der Run
|
|||||||
5. Release veröffentlichen; Gitea erzeugt dabei den Git-Tag auf dem konkreten Commit.
|
5. Release veröffentlichen; Gitea erzeugt dabei den Git-Tag auf dem konkreten Commit.
|
||||||
6. `latest` und `main` auf die veröffentlichten Images umstellen, anschließend die Sperre entfernen.
|
6. `latest` und `main` auf die veröffentlichten Images umstellen, anschließend die Sperre entfernen.
|
||||||
|
|
||||||
Bei einem Fehler den fehlgeschlagenen Workflow **am ursprünglichen Commit erneut ausführen**. Ein vorhandenes Manifest wird weiterverwendet; veröffentlichte Versionen werden nicht neu gebaut. Auch eine unterbrochene Aktualisierung von `latest` lässt sich so vervollständigen. Die Registry bietet keine gemeinsame Transaktion über mehrere Images: deshalb verwendet Selfhosting feste Versionen und das Manifest die exakten Digests.
|
Bei einem Release-Fehler den fehlgeschlagenen Workflow **am ursprünglichen Commit erneut ausführen**. Ein vorhandenes Manifest wird weiterverwendet; veröffentlichte Versionen werden nicht neu gebaut. Auch eine unterbrochene Aktualisierung von `latest` lässt sich so vervollständigen. Die Registry bietet keine gemeinsame Transaktion über mehrere Images: deshalb verwendet Selfhosting feste Versionen und das Manifest die exakten Digests.
|
||||||
|
|
||||||
Ein anderer noch offener Release-Entwurf blockiert einen neuen Release. Zuerst den ursprünglichen Lauf wiederaufnehmen; anschließend den aktuellen Main-Lauf erneut starten. Releases oder Versionstags nicht löschen, um einen fehlgeschlagenen Lauf zu überspringen.
|
Ein anderer noch offener Release-Entwurf blockiert einen neuen Release. Zuerst den ursprünglichen Lauf wiederaufnehmen; anschließend den aktuellen Main-Lauf erneut starten. Releases oder Versionstags nicht löschen, um einen fehlgeschlagenen Lauf zu überspringen.
|
||||||
|
|
||||||
@@ -103,7 +109,7 @@ git ls-remote origin refs/heads/automation/release-lock
|
|||||||
git push --force-with-lease=refs/heads/automation/release-lock:LOCK_SHA origin :refs/heads/automation/release-lock
|
git push --force-with-lease=refs/heads/automation/release-lock:LOCK_SHA origin :refs/heads/automation/release-lock
|
||||||
```
|
```
|
||||||
|
|
||||||
Für einen abgebrochenen Entwicklungsbuild entsprechend `automation/dev-build-lock` verwenden. Keine aktive Sperre löschen. Der Lease verhindert, dass zwischenzeitlich eine andere Sperre entfernt wird.
|
Keine aktive Sperre löschen. Der Lease verhindert, dass zwischenzeitlich eine andere Sperre entfernt wird. Entwicklungsbuilds erzeugen keine Sperre und können direkt erneut gestartet werden.
|
||||||
|
|
||||||
## Selfhosting und Updates
|
## Selfhosting und Updates
|
||||||
|
|
||||||
@@ -116,7 +122,9 @@ bash /tmp/fedeo-install.sh --simple --start
|
|||||||
|
|
||||||
Dieser versionierte Installer lädt die Betriebsdateien vom zugehörigen Git-Tag und schreibt `FEDEO_VERSION=3.0.0` in die neue `.env`. Vorhandene Installationen behalten ihren bisherigen Kanal, bis `FEDEO_VERSION` bewusst geändert wird. Die Dateien im Entwicklungszweig verwenden weiterhin `dev` als Rückfallwert.
|
Dieser versionierte Installer lädt die Betriebsdateien vom zugehörigen Git-Tag und schreibt `FEDEO_VERSION=3.0.0` in die neue `.env`. Vorhandene Installationen behalten ihren bisherigen Kanal, bis `FEDEO_VERSION` bewusst geändert wird. Die Dateien im Entwicklungszweig verwenden weiterhin `dev` als Rückfallwert.
|
||||||
|
|
||||||
Bei einem Update:
|
Der vollständige Host-Updater einschließlich Compose-Austausch und Backup ist in [Vollständige Selfhost-Updates](selfhost-updates.md) beschrieben. Er benötigt eine Zielversion mit gepinntem MinIO und Anwendungs-Healthchecks.
|
||||||
|
|
||||||
|
Manueller Ablauf für ältere Releases oder abweichende Installationen:
|
||||||
|
|
||||||
1. Release Notes und Migrationshinweise lesen. Konsistentes Datenbankbackup (z. B. `pg_dump`) sowie Datei-/Objektspeicher und Konfiguration sichern. Ein TAR des laufenden PostgreSQL-Datenverzeichnisses ist kein verlässliches Datenbankbackup.
|
1. Release Notes und Migrationshinweise lesen. Konsistentes Datenbankbackup (z. B. `pg_dump`) sowie Datei-/Objektspeicher und Konfiguration sichern. Ein TAR des laufenden PostgreSQL-Datenverzeichnisses ist kein verlässliches Datenbankbackup.
|
||||||
2. Die benötigten Betriebsdateien vom gewünschten Release herunterladen; bestehende `.env` und Daten behalten. Downloads mit dessen `SHA256SUMS` überprüfen.
|
2. Die benötigten Betriebsdateien vom gewünschten Release herunterladen; bestehende `.env` und Daten behalten. Downloads mit dessen `SHA256SUMS` überprüfen.
|
||||||
|
|||||||
98
docs/selfhost-updates.md
Normal file
98
docs/selfhost-updates.md
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
# Vollständige Selfhost-Updates
|
||||||
|
|
||||||
|
`scripts/selfhost-update.py` ist der gemeinsame Update-Ablauf für die Kommandozeile und einen zukünftigen Update-Dienst. Diese Implementierung installiert noch keinen Dienst und stellt noch keinen Update-Button bereit. Der Prozess muss auf dem Docker-Host außerhalb der FEDEO-Container laufen.
|
||||||
|
|
||||||
|
## Voraussetzungen und Umfang
|
||||||
|
|
||||||
|
- Linux/amd64, Python ab 3.10, Docker Compose ab 2.24, `tar` und `du`.
|
||||||
|
- Bestehendes Betriebsverzeichnis, standardmäßig `/opt/fedeo`, mit `.env` und `docker-compose.yml`; kein Git-Checkout.
|
||||||
|
- Standard-Stack mit `traefik`, `db`, `minio`, `createbuckets`, `backend`, `frontend` und `node-exporter`. PostgreSQL und MinIO verwenden die lokalen Verzeichnisse `postgres/` und `minio/`.
|
||||||
|
- Als root auf dem Server ausführen: Docker-Zugriff und Leserechte für die Daten- und Zertifikatsverzeichnisse werden benötigt. Ein entfernter Docker-Daemon wird abgewiesen.
|
||||||
|
- Die öffentliche HTTPS-Adresse muss vom Host erreichbar sein und ein gültiges Zertifikat haben.
|
||||||
|
- Eine Wartungsunterbrechung für alle Mandanten einplanen. Vor dem Backup werden alle laufenden Dienste außer PostgreSQL angehalten, auch der öffentliche Proxy und alle im Backend laufenden Hintergrundaufgaben. Eine separate Wartungsseite folgt mit der späteren Oberflächenintegration.
|
||||||
|
|
||||||
|
Der Updater verweigert externe Datenbanken, externen S3-/Seafile-Speicher, zusätzliche Dienste, Compose-Profile, geänderte Datenpfade und PostgreSQL-Imagewechsel. Dafür braucht es zunächst passende Backup-/Migrationsadapter. Eigene Einstellungen innerhalb der unterstützten Dienste können in `docker-compose.override.yml` **oder** `docker-compose.override.yaml` bleiben; der Updater verwendet diese Datei bei jedem Compose-Aufruf. Änderungen an dieser Override-Datei werden nicht überschrieben.
|
||||||
|
|
||||||
|
Die Zielversion muss bereits die festen MinIO-Digests und die neuen Backend-/Frontend-Healthchecks enthalten. Ältere Releases bleiben über den in `docs/releases.md` beschriebenen manuellen Ablauf installierbar. Ein Versionswechsel des Betriebssystems oder der Docker Engine gehört nicht zu diesem Ablauf.
|
||||||
|
|
||||||
|
## Bereitstellung
|
||||||
|
|
||||||
|
Neue versionierte Installer laden das Skript und diese Anleitung bereits ins Betriebsverzeichnis und berücksichtigen Python 3 bei den Basispaketen. Bestehende Installationen können `selfhost-update.py` und `SELFHOST-UPDATES.md` aus den Assets eines entsprechenden veröffentlichten Releases übernehmen. Zuerst mit dessen `SHA256SUMS` prüfen, anschließend das Skript unter `/opt/fedeo/scripts/selfhost-update.py` ablegen. Für diese erstmalige Bereitstellung ist noch ein Serverzugriff erforderlich.
|
||||||
|
|
||||||
|
Downloads erfolgen ausschließlich vom konfigurierten FEDEO-Repository über HTTPS. Die Dateiprüfsummen werden gegen `SHA256SUMS`, das Manifest gegen den Git-Tag geprüft. Die Prüfsummen sind **keine unabhängige Signatur**; das Vertrauen liegt beim Gitea-Server und dessen Release-Veröffentlichung. Falls das Repository privat ist, `FEDEO_RELEASE_TOKEN` mit Leserechten nur für den Aufruf setzen. Registry-Anmeldung bei Bedarf vorher über `docker login git.federspiel.tech` im Konto des Updaters durchführen. Der Updater speichert den Release-Token nicht in Statusdateien und gibt ihn bei Weiterleitungen an andere Hosts nicht weiter.
|
||||||
|
|
||||||
|
## Prüfen, anwenden und Status lesen
|
||||||
|
|
||||||
|
Neueste stabile Version suchen oder eine bestimmte Version prüfen:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py check
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py check --version 3.1.0
|
||||||
|
```
|
||||||
|
|
||||||
|
`3.1.0` ist ein Beispiel und muss durch einen tatsächlich veröffentlichten Zielrelease ersetzt werden. Ein anderes Betriebsverzeichnis wird mit `--directory /pfad/zur/instanz` angegeben.
|
||||||
|
|
||||||
|
Die Prüfung lädt die Betriebsdateien und erstellt einen Plan einschließlich `compose.diff` und Release Notes unter `/opt/fedeo/.updates/<Auftrag>/`. Sie verändert die laufende Installation nicht und lädt noch keine Docker-Images. Die JSON-Ausgabe nennt Zielversion, Commit, betroffene Dienste, geschätzten Backup-Platz und die Pfade der Vorschau. Unveröffentlichte Entwürfe und Vorabversionen werden ignoriert.
|
||||||
|
|
||||||
|
Beim ersten Update kennt der Updater die Herkunft der vorhandenen Compose-Datei noch nicht. Die erzeugte `compose.diff` prüfen und eigene Anpassungen in die Override-Datei übertragen. Dann den Austausch ausdrücklich akzeptieren:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py apply --version 3.1.0 --accept-compose-replacement
|
||||||
|
```
|
||||||
|
|
||||||
|
Für spätere Updates unveränderter, vom Updater verwalteter Compose-Dateien:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py apply --version 3.2.0
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py status
|
||||||
|
```
|
||||||
|
|
||||||
|
Eine explizite Zielversion ist beim Anwenden erforderlich. `apply` prüft die Release-Dateien und die tatsächliche Installation nochmals. Bei gleichen oder älteren stabilen Zielversionen wird abgebrochen. Änderungen an `.env`, Basis-Compose oder der Override-Datei während des Downloads führen ebenfalls vor der Wartungsphase zum Abbruch.
|
||||||
|
|
||||||
|
Während des ersten Ausbaus den Befehl in einer bestehenden persistenten Terminalsitzung ausführen. Der spätere Host-Dienst übernimmt die vom Browser unabhängige Ausführung. Ein Neustart des FEDEO-Stacks beendet diesen Host-Prozess nicht; ein Neustart des Hosts unterbricht ihn.
|
||||||
|
|
||||||
|
## Ablauf und Speicherung
|
||||||
|
|
||||||
|
1. Exklusive Host-Sperre erwerben; offene Wiederherstellungsfälle blockieren neue Updates.
|
||||||
|
2. Release samt Prüfsummen laden, Manifest/Commit prüfen und die FEDEO-Images auf die exakten Digests setzen.
|
||||||
|
3. Aktuelle und neue Compose-Konfiguration mit Docker Compose validieren. Projektname, Container-Identitäten, Mounts, Netzwerke und Zugangsdaten erhalten. Die Shell-Umgebung überschreibt dabei keine Werte aus der Instanz-`.env`.
|
||||||
|
4. Speicherplatz prüfen: mindestens 5 GiB frei am Docker-Datenverzeichnis vor dem Pull sowie geschätzte Backup-Größe plus Reserve im Betriebsverzeichnis. Images laden, Backup-Platz erneut prüfen.
|
||||||
|
5. Schreibende Dienste und Proxy anhalten. PostgreSQL bleibt für `pg_dump` aktiv.
|
||||||
|
6. Konfiguration, Container-Metadaten, PostgreSQL-Dump im Custom-Format, globale PostgreSQL-Rollen, gestoppten MinIO-Dateispeicher und Zertifikate sichern. Dump und TAR auf Lesbarkeit prüfen und SHA-256-Prüfsummen schreiben. `backup/complete.json` markiert ein vollständig geschriebenes Backup; ein Restore-Probelauf auf einer getrennten Instanz wird dadurch nicht ersetzt.
|
||||||
|
7. Compose-Datei und `.env` einzeln atomar ersetzen. In `.env` wird ausschließlich `FEDEO_VERSION` verändert; `.env.example` und Setup-Skript werden erneuert. Den laufenden Updater ersetzt dieser Vorgang nicht.
|
||||||
|
8. Infrastruktur starten, Bucket-Einrichtung ausführen und Migrationen explizit in einem benannten Einmalcontainer starten. Beim anschließenden regulären Backend-Start läuft dessen bestehende Migrationsprüfung nochmals; bereits angewendete Migrationen werden dabei übersprungen.
|
||||||
|
9. Stack mit `up --wait` starten. Backend-/Frontend-Image-IDs und Healthchecks sowie öffentlich gemeldete Version und Commit prüfen. Erst dann den Release als installiert speichern.
|
||||||
|
|
||||||
|
Status und Phasen werden atomar in `.updates/status.json` und pro Auftrag in `status.json` gespeichert. `events.jsonl` enthält den Verlauf, `commands.log` Fehlermeldungen der Werkzeuge. Das Verzeichnis hat Modus `0700`; Status- und Konfigurationsdateien werden mit `0600` angelegt. Backups und Werkzeugausgaben enthalten vertrauliche Daten und dürfen nicht ungefiltert an eine Oberfläche ausgeliefert werden. Backups werden nicht automatisch gelöscht.
|
||||||
|
|
||||||
|
## Fehler und Wiederherstellung
|
||||||
|
|
||||||
|
- **Download, Konfiguration oder Pull fehlgeschlagen:** Der laufende Stack bleibt bestehen.
|
||||||
|
- **Backup oder Stop fehlgeschlagen:** Der Updater versucht, die zuvor laufenden Container wieder zu starten. Gelingt das nicht, bleibt der Wiederherstellungsstatus gesetzt.
|
||||||
|
- **Ab Austausch der Dateien/Infrastruktur:** Keine automatische Rückkehr zu älteren Images. Bei einem Fehler werden Backend, Frontend und Proxy nach Möglichkeit angehalten. Das Datenbankschema oder das Speicherformat könnten bereits verändert sein.
|
||||||
|
- **Prozess hart beendet oder Host neu gestartet:** `status` erkennt einen zurückgelassenen laufenden Auftrag als `interrupted`. Eine vor Beginn der Wartung gespeicherte Wiederherstellungssperre bleibt erhalten. Abgeschlossene Teiloperationen werden nicht als erfolgreiches Update ausgegeben.
|
||||||
|
|
||||||
|
Zur Diagnose `status` und die geschützten Auftragsdateien lesen. Falls ein benannter `fedeo-update-<Auftrag>`-Migrationscontainer zurückgeblieben ist, dessen Zustand und Logs zuerst prüfen. Er kann unabhängig vom abgebrochenen Compose-Prozess weiterlaufen; erst nach geklärtem Zustand anhalten/entfernen.
|
||||||
|
|
||||||
|
Für die Rückkehr zum alten Stand in einem geplanten Wartungsfenster:
|
||||||
|
|
||||||
|
1. Alle schreibenden Dienste und den Proxy angehalten lassen; keinen neuen Update-Auftrag starten.
|
||||||
|
2. Prüfsummen aus `backup/complete.json` prüfen. Ursprüngliche Compose-Datei, `.env` und gegebenenfalls Override aus `backup/config/` wiederherstellen. `backup/containers.json` dokumentiert die ursprünglichen Image-IDs; insbesondere bewegliche Infrastruktur-Tags vor dem Start auf die gesicherten Image-Stände festlegen.
|
||||||
|
3. Falls Migrationen/Infrastrukturänderungen bereits begonnen haben: PostgreSQL und Dateispeicher aus dem **zusammengehörigen** Backup wiederherstellen. Der PostgreSQL-Dump ist für `pg_restore` geeignet; `globals.sql` enthält Rollen und Kennwörter und muss beim Wiederaufbau der Datenbank berücksichtigt werden. MinIO und Zertifikate bei gestoppten Diensten aus `files.tar` wiederherstellen. Die konkrete Rücksicherung bestehender Daten erfolgt bewusst manuell und ersetzt Daten.
|
||||||
|
4. Wiederhergestellten Stack mit dessen Compose-Dateien starten und Funktionen prüfen. Alternativ den Fehler der Zielversion beheben und diese erfolgreich starten.
|
||||||
|
5. Erst nach erfolgreicher Wiederherstellung die Sperre aufheben:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo python3 /opt/fedeo/scripts/selfhost-update.py resolve --version 3.0.0
|
||||||
|
```
|
||||||
|
|
||||||
|
`resolve` prüft die angegebene Backend-Version, Backend-Erreichbarkeit, Frontend-Erreichbarkeit, laufende Kerndienste, passende Anwendungs-Images und das Fehlen eines zurückgebliebenen Migrationscontainers. Es stellt selbst **keine Daten wieder her**. Bei alten Entwicklungsinstallationen die tatsächlich von `/backend/version` gemeldete Entwicklungsversion angeben. Ein späteres Update kann nach einer manuellen Compose-Wiederherstellung erneut `--accept-compose-replacement` erfordern.
|
||||||
|
|
||||||
|
## Entwicklung und Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm test
|
||||||
|
npm run commits:check
|
||||||
|
```
|
||||||
|
|
||||||
|
Die Update-Tests verwenden den echten Compose-Parser und einen simulierten Docker-Daemon. Sie prüfen u. a. Erfolg, Prüfsummen, Release-Auswahl, unveränderte Konfiguration, Speicherplatz, parallele Aufträge, Backupfehler, Migrationsfehler, falsche laufende Versionen und unterbrochene Aufträge. Ein vollständiger Test mit realem PostgreSQL-/MinIO-Restore auf einer separaten Selfhost-Testinstanz ist vor dem Produktionseinsatz zusätzlich erforderlich.
|
||||||
10
frontend/server/api/health.get.ts
Normal file
10
frontend/server/api/health.get.ts
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
export default defineEventHandler((event) => {
|
||||||
|
setHeader(event, 'Cache-Control', 'no-store')
|
||||||
|
const config = useRuntimeConfig(event)
|
||||||
|
return {
|
||||||
|
status: 'ok',
|
||||||
|
product: 'FEDEO',
|
||||||
|
version: config.public.fedeoVersion,
|
||||||
|
revision: config.public.fedeoRevision,
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -4,7 +4,8 @@
|
|||||||
"type": "module",
|
"type": "module",
|
||||||
"engines": { "node": ">=22" },
|
"engines": { "node": ">=22" },
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "node --test scripts/release/tests/*.test.mjs",
|
"test": "node --test scripts/release/tests/*.test.mjs && npm run test:update",
|
||||||
|
"test:update": "python3 -B -m unittest discover -s scripts/update/tests -v",
|
||||||
"commits:check": "node scripts/release/cli.mjs check",
|
"commits:check": "node scripts/release/cli.mjs check",
|
||||||
"release:preview": "node scripts/release/cli.mjs preview",
|
"release:preview": "node scripts/release/cli.mjs preview",
|
||||||
"release:ci": "node scripts/release/ci.mjs"
|
"release:ci": "node scripts/release/ci.mjs"
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
{
|
{
|
||||||
"initialVersion": "3.0.0",
|
"initialVersion": "3.0.0",
|
||||||
"bootstrapCommit": "cfef07b96d97f8dffdfc32c74f8f32b62fe43b18",
|
"bootstrapCommit": "cfef07b96d97f8dffdfc32c74f8f32b62fe43b18",
|
||||||
|
"legacyHistoryHeads": ["e7554fa2cc649f69dce4880abe76c6fab0f82700"],
|
||||||
"branch": "main",
|
"branch": "main",
|
||||||
"repositoryUrl": "https://git.federspiel.tech/flfeders/FEDEO",
|
"repositoryUrl": "https://git.federspiel.tech/flfeders/FEDEO",
|
||||||
"registry": "git.federspiel.tech/flfeders/fedeo",
|
"registry": "git.federspiel.tech/flfeders/fedeo",
|
||||||
|
|||||||
@@ -39,16 +39,10 @@ async function promote(image, tags, immutable) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function buildImages(plan, version, reuseTag) {
|
async function buildImages(plan, version, stagingTag = `build-${randomUUID()}`) {
|
||||||
const stagingTag = `build-${randomUUID()}`;
|
|
||||||
const images = [];
|
const images = [];
|
||||||
for (const component of config.images) {
|
for (const component of config.images) {
|
||||||
const repository = `${config.registry}/${component.name}`;
|
const repository = `${config.registry}/${component.name}`;
|
||||||
const existing = reuseTag ? imageDigest(`${repository}:${reuseTag}`, true) : null;
|
|
||||||
if (existing) {
|
|
||||||
images.push({ name: component.name, repository, digest: existing });
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const metadata = `${outputDir}/${component.name}.json`;
|
const metadata = `${outputDir}/${component.name}.json`;
|
||||||
console.log(`Baue ${component.name} ${version} (${plan.sha.slice(0, 12)})`);
|
console.log(`Baue ${component.name} ${version} (${plan.sha.slice(0, 12)})`);
|
||||||
docker(['buildx', 'build', '--push', '--platform', 'linux/amd64', '--provenance=false',
|
docker(['buildx', 'build', '--push', '--platform', 'linux/amd64', '--provenance=false',
|
||||||
@@ -71,15 +65,15 @@ try {
|
|||||||
}
|
}
|
||||||
const sha = git(['rev-parse', 'HEAD']);
|
const sha = git(['rev-parse', 'HEAD']);
|
||||||
if (sha !== process.env.FEDEO_SHA) throw new Error('Checkout stimmt nicht mit dem Workflow-Commit überein.');
|
if (sha !== process.env.FEDEO_SHA) throw new Error('Checkout stimmt nicht mit dem Workflow-Commit überein.');
|
||||||
const token = process.env.GITEA_TOKEN;
|
|
||||||
if (!token) throw new Error('CI_TOKEN oder RELEASE_TOKEN als Gitea-Secret konfigurieren (siehe docs/releases.md).');
|
|
||||||
mkdirSync(outputDir, { recursive: true });
|
|
||||||
const api = new Gitea(config.repositoryUrl, token);
|
|
||||||
const stable = ref === `refs/heads/${config.branch}`;
|
const stable = ref === `refs/heads/${config.branch}`;
|
||||||
unlock = await acquireLock(stable ? {} : { ref: 'refs/heads/automation/dev-build-lock' });
|
const token = process.env.GITEA_TOKEN;
|
||||||
|
if (stable && !token) throw new Error('Für main einen RELEASE_TOKEN oder CI_TOKEN mit write:repository konfigurieren (siehe docs/releases.md).');
|
||||||
|
mkdirSync(outputDir, { recursive: true });
|
||||||
|
if (stable) unlock = await acquireLock();
|
||||||
git(['fetch', 'origin', `refs/heads/${config.branch}:refs/remotes/origin/${config.branch}`, '--tags']);
|
git(['fetch', 'origin', `refs/heads/${config.branch}:refs/remotes/origin/${config.branch}`, '--tags']);
|
||||||
|
|
||||||
if (stable) {
|
if (stable) {
|
||||||
|
const api = new Gitea(config.repositoryUrl, token);
|
||||||
const releases = await api.releases();
|
const releases = await api.releases();
|
||||||
const pendingForThisCommit = releases.some(release => release.draft && stableTag.test(release.tag_name) && release.target_commitish === sha);
|
const pendingForThisCommit = releases.some(release => release.draft && stableTag.test(release.tag_name) && release.target_commitish === sha);
|
||||||
if (git(['rev-parse', `origin/${config.branch}`]) !== sha && !pendingForThisCommit) {
|
if (git(['rev-parse', `origin/${config.branch}`]) !== sha && !pendingForThisCommit) {
|
||||||
@@ -96,10 +90,19 @@ try {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const plan = await createPlan();
|
const plan = await createPlan();
|
||||||
const images = await buildImages(plan, plan.developmentVersion, `dev-sha-${sha}`);
|
// Separate snapshots let overlapping dev runs build without a repository write lock.
|
||||||
for (const image of images) await promote(image, [`dev-sha-${sha}`], true);
|
const snapshotTag = `dev-sha-${sha}-${randomUUID()}`;
|
||||||
const remoteDev = git(['ls-remote', 'origin', 'refs/heads/dev']).split(/\s/)[0];
|
const images = await buildImages(plan, plan.developmentVersion, snapshotTag);
|
||||||
if (remoteDev === sha) for (const image of images) await promote(image, ['dev'], false);
|
console.log(`Entwicklungsbuild: ${snapshotTag}`);
|
||||||
|
for (const image of images) {
|
||||||
|
// Check each update so an outdated run stops moving the mutable dev channel.
|
||||||
|
const remoteDev = git(['ls-remote', 'origin', 'refs/heads/dev']).split(/\s/)[0];
|
||||||
|
if (remoteDev !== sha) {
|
||||||
|
console.log('Ein neuerer dev-Commit liegt vor; der dev-Kanal wird nicht weiter aktualisiert.');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
await promote(image, ['dev'], false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error.message);
|
console.error(error.message);
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ try {
|
|||||||
if (command === 'check') {
|
if (command === 'check') {
|
||||||
const config = readConfig();
|
const config = readConfig();
|
||||||
assertBootstrap(config, 'HEAD', process.cwd());
|
assertBootstrap(config, 'HEAD', process.cwd());
|
||||||
const commits = commitsSince(config.bootstrapCommit);
|
const commits = commitsSince(config.bootstrapCommit, 'HEAD', process.cwd(), config.legacyHistoryHeads);
|
||||||
checkCommits(commits);
|
checkCommits(commits);
|
||||||
console.log(`${commits.length} Commit-Nachrichten geprüft.`);
|
console.log(`${commits.length} Commit-Nachrichten geprüft.`);
|
||||||
} else if (command === 'preview') {
|
} else if (command === 'preview') {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { execFileSync } from 'node:child_process';
|
import { execFileSync, spawn } from 'node:child_process';
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
import { readFileSync } from 'node:fs';
|
import { readFileSync } from 'node:fs';
|
||||||
import { resolve } from 'node:path';
|
import { resolve } from 'node:path';
|
||||||
@@ -24,9 +24,9 @@ export function validCommit(message) {
|
|||||||
(/^Revert ".+"$/.test(message.split('\n')[0]) && /^This reverts commit [a-f0-9]{40}\.$/m.test(message));
|
(/^Revert ".+"$/.test(message.split('\n')[0]) && /^This reverts commit [a-f0-9]{40}\.$/m.test(message));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function commitsSince(from, to = 'HEAD', cwd = process.cwd()) {
|
export function commitsSince(from, to = 'HEAD', cwd = process.cwd(), excludedHeads = []) {
|
||||||
// semantic-release's revert filter expects newest commits first.
|
// semantic-release's revert filter expects newest commits first.
|
||||||
const hashes = git(['rev-list', '--no-merges', `${from}..${to}`], cwd);
|
const hashes = git(['rev-list', '--no-merges', `${from}..${to}`, ...excludedHeads.map(head => `^${head}`)], cwd);
|
||||||
return hashes ? hashes.split('\n').map(hash => ({ hash, message: git(['show', '-s', '--format=%B', hash], cwd) })) : [];
|
return hashes ? hashes.split('\n').map(hash => ({ hash, message: git(['show', '-s', '--format=%B', hash], cwd) })) : [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,6 +42,11 @@ export async function releaseType(commits, cwd = process.cwd()) {
|
|||||||
export function assertBootstrap(config, target, cwd) {
|
export function assertBootstrap(config, target, cwd) {
|
||||||
try { git(['merge-base', '--is-ancestor', config.bootstrapCommit, target], cwd); }
|
try { git(['merge-base', '--is-ancestor', config.bootstrapCommit, target], cwd); }
|
||||||
catch { throw new Error('Der Release-Ausgangscommit fehlt in der Historie. Vollständig auschecken (fetch-depth: 0) und dev nach main übernehmen.'); }
|
catch { throw new Error('Der Release-Ausgangscommit fehlt in der Historie. Vollständig auschecken (fetch-depth: 0) und dev nach main übernehmen.'); }
|
||||||
|
for (const head of config.legacyHistoryHeads ?? []) {
|
||||||
|
if (!/^[a-f0-9]{40}$/.test(head)) throw new Error('legacyHistoryHeads muss feste vollständige Commit-IDs enthalten.');
|
||||||
|
try { git(['rev-parse', '--verify', `${head}^{commit}`], cwd); }
|
||||||
|
catch { throw new Error(`Die alte Release-Historie ${head} fehlt. Alle Branches vollständig laden (fetch-depth: 0).`); }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd), target = 'HEAD' } = {}) {
|
export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd), target = 'HEAD' } = {}) {
|
||||||
@@ -53,8 +58,11 @@ export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd)
|
|||||||
const resume = !!lastTag && git(['rev-parse', `${lastTag}^{commit}`], cwd) === sha;
|
const resume = !!lastTag && git(['rev-parse', `${lastTag}^{commit}`], cwd) === sha;
|
||||||
const previousTag = resume ? tags[1] : lastTag;
|
const previousTag = resume ? tags[1] : lastTag;
|
||||||
const from = previousTag ? git(['rev-parse', `${previousTag}^{commit}`], cwd) : config.bootstrapCommit;
|
const from = previousTag ? git(['rev-parse', `${previousTag}^{commit}`], cwd) : config.bootstrapCommit;
|
||||||
const commits = commitsSince(from, sha, cwd);
|
// Old main-only commits are not ancestors of the dev bootstrap. Exclude their
|
||||||
checkCommits(commitsSince(config.bootstrapCommit, sha, cwd));
|
// pinned history without dropping new work branched from those old commits.
|
||||||
|
const legacyHeads = config.legacyHistoryHeads ?? [];
|
||||||
|
const commits = commitsSince(from, sha, cwd, legacyHeads);
|
||||||
|
checkCommits(commitsSince(config.bootstrapCommit, sha, cwd, legacyHeads));
|
||||||
const bump = await releaseType(commits, cwd);
|
const bump = await releaseType(commits, cwd);
|
||||||
const version = resume ? lastTag.slice(1) : !lastTag ? config.initialVersion : bump ? semver.inc(lastTag.slice(1), bump) : null;
|
const version = resume ? lastTag.slice(1) : !lastTag ? config.initialVersion : bump ? semver.inc(lastTag.slice(1), bump) : null;
|
||||||
if (version && !semver.valid(version)) throw new Error('Ungültige Release-Version.');
|
if (version && !semver.valid(version)) throw new Error('Ungültige Release-Version.');
|
||||||
@@ -67,7 +75,7 @@ export async function createPlan({ cwd = process.cwd(), config = readConfig(cwd)
|
|||||||
lastRelease: previousTag ? { version: previousTag.slice(1), gitTag: previousTag, gitHead: from } : {},
|
lastRelease: previousTag ? { version: previousTag.slice(1), gitTag: previousTag, gitHead: from } : {},
|
||||||
nextRelease: { version, gitTag: tag, gitHead: sha },
|
nextRelease: { version, gitTag: tag, gitHead: sha },
|
||||||
}) : '';
|
}) : '';
|
||||||
const count = git(['rev-list', '--count', `${config.bootstrapCommit}..${sha}`], cwd);
|
const count = git(['rev-list', '--count', `${config.bootstrapCommit}..${sha}`, ...legacyHeads.map(head => `^${head}`)], cwd);
|
||||||
return {
|
return {
|
||||||
sha, version, tag, previousTag: previousTag || null, from, bump, resume,
|
sha, version, tag, previousTag: previousTag || null, from, bump, resume,
|
||||||
developmentVersion: `${lastTag ? lastTag.slice(1) : '0.0.0'}-dev.${count}.g${sha.slice(0, 12)}`,
|
developmentVersion: `${lastTag ? lastTag.slice(1) : '0.0.0'}-dev.${count}.g${sha.slice(0, 12)}`,
|
||||||
@@ -83,11 +91,33 @@ export function componentVersions(sha, cwd = process.cwd()) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function migrationManifest(sha, cwd = process.cwd()) {
|
function gitBlobSha256(reference, cwd) {
|
||||||
return ['backend/db/migrations', 'push-server/packages/db/drizzle'].flatMap(folder => {
|
return new Promise((resolve, reject) => {
|
||||||
const files = git(['ls-tree', '-r', '--name-only', sha, '--', folder], cwd);
|
const child = spawn('git', ['cat-file', 'blob', reference], { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||||
return files ? files.split('\n').filter(path => path.endsWith('.sql')).map(path => ({
|
const hash = createHash('sha256');
|
||||||
path, sha256: createHash('sha256').update(execFileSync('git', ['show', `${sha}:${path}`], { cwd })).digest('hex'),
|
let stderr = '';
|
||||||
})) : [];
|
const fail = message => reject(new Error(`Prüfsumme für ${reference} fehlgeschlagen: ${message}`));
|
||||||
|
// Reference-data migrations can exceed 90 MB. Hash raw chunks without
|
||||||
|
// buffering the entire blob or altering whitespace/encoding.
|
||||||
|
child.stdout.on('data', chunk => hash.update(chunk));
|
||||||
|
child.stderr.setEncoding('utf8');
|
||||||
|
child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(0, 8192); });
|
||||||
|
child.once('error', error => fail(error.message));
|
||||||
|
for (const stream of [child.stdout, child.stderr]) {
|
||||||
|
stream.once('error', error => { child.kill(); fail(error.message); });
|
||||||
|
}
|
||||||
|
child.once('close', (code, signal) => {
|
||||||
|
if (code !== 0) fail(`git cat-file (Exit ${code ?? signal}): ${stderr.trim()}`);
|
||||||
|
else resolve(hash.digest('hex'));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function migrationManifest(sha, cwd = process.cwd()) {
|
||||||
|
const files = git(['ls-tree', '-r', '-z', '--name-only', sha, '--',
|
||||||
|
'backend/db/migrations', 'push-server/packages/db/drizzle'], cwd)
|
||||||
|
.split('\0').filter(path => path.endsWith('.sql'));
|
||||||
|
const manifest = [];
|
||||||
|
for (const path of files) manifest.push({ path, sha256: await gitBlobSha256(`${sha}:${path}`, cwd) });
|
||||||
|
return manifest;
|
||||||
|
}
|
||||||
|
|||||||
@@ -15,6 +15,9 @@ export async function acquireLock({ runGit = git, wait = sleep, attempts = 480,
|
|||||||
runGit(['push', `--force-with-lease=${ref}:`, 'origin', `${lockSha}:${ref}`]);
|
runGit(['push', `--force-with-lease=${ref}:`, 'origin', `${lockSha}:${ref}`]);
|
||||||
return () => runGit(['push', `--force-with-lease=${ref}:${lockSha}`, 'origin', `:${ref}`]);
|
return () => runGit(['push', `--force-with-lease=${ref}:${lockSha}`, 'origin', `:${ref}`]);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (/403|forbidden|permission denied|authentication failed/i.test(String(error.stderr || error.message))) {
|
||||||
|
throw new Error('Die main-Release-Sperre benötigt Git-Schreibrechte. RELEASE_TOKEN (Secret oder Variable) oder den bisherigen vars.CI_TOKEN mit write:repository und Zugriff auf automation/release-lock konfigurieren.');
|
||||||
|
}
|
||||||
if (!runGit(['ls-remote', 'origin', ref])) throw error;
|
if (!runGit(['ls-remote', 'origin', ref])) throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,11 +14,11 @@ export function assertManifest(manifest, plan, config) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function makeManifest(plan, images, config) {
|
export async function makeManifest(plan, images, config) {
|
||||||
return {
|
return {
|
||||||
schemaVersion: 1, version: plan.version, tag: plan.tag, commit: plan.sha,
|
schemaVersion: 1, version: plan.version, tag: plan.tag, commit: plan.sha,
|
||||||
previousTag: plan.previousTag, repository: config.repositoryUrl,
|
previousTag: plan.previousTag, repository: config.repositoryUrl,
|
||||||
images, components: componentVersions(plan.sha), migrations: migrationManifest(plan.sha),
|
images, components: componentVersions(plan.sha), migrations: await migrationManifest(plan.sha),
|
||||||
migrationPolicy: 'Forward migrations on startup. Downgrade requires schema compatibility or a matching database backup.',
|
migrationPolicy: 'Forward migrations on startup. Downgrade requires schema compatibility or a matching database backup.',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -32,6 +32,8 @@ export function releaseFiles(manifest, notes) {
|
|||||||
['.env.example', Buffer.from(readFileSync('.env.example', 'utf8').replace(/^FEDEO_VERSION=.*$/m, `FEDEO_VERSION=${manifest.version}`))],
|
['.env.example', Buffer.from(readFileSync('.env.example', 'utf8').replace(/^FEDEO_VERSION=.*$/m, `FEDEO_VERSION=${manifest.version}`))],
|
||||||
['selfhost-install.sh', Buffer.from(readFileSync('scripts/selfhost-install.sh', 'utf8').replace('FEDEO_REF:-dev', `FEDEO_REF:-${manifest.tag}`))],
|
['selfhost-install.sh', Buffer.from(readFileSync('scripts/selfhost-install.sh', 'utf8').replace('FEDEO_REF:-dev', `FEDEO_REF:-${manifest.tag}`))],
|
||||||
['selfhost-setup.sh', Buffer.from(readFileSync('scripts/selfhost-setup.sh', 'utf8').replaceAll('FEDEO_VERSION:-dev', `FEDEO_VERSION:-${manifest.version}`))],
|
['selfhost-setup.sh', Buffer.from(readFileSync('scripts/selfhost-setup.sh', 'utf8').replaceAll('FEDEO_VERSION:-dev', `FEDEO_VERSION:-${manifest.version}`))],
|
||||||
|
['selfhost-update.py', Buffer.from(readFileSync('scripts/selfhost-update.py', 'utf8'))],
|
||||||
|
['SELFHOST-UPDATES.md', Buffer.from(readFileSync('docs/selfhost-updates.md', 'utf8'))],
|
||||||
]);
|
]);
|
||||||
files.set('SHA256SUMS', Buffer.from([...files].map(([name, data]) => `${createHash('sha256').update(data).digest('hex')} ${name}`).join('\n') + '\n'));
|
files.set('SHA256SUMS', Buffer.from([...files].map(([name, data]) => `${createHash('sha256').update(data).digest('hex')} ${name}`).join('\n') + '\n'));
|
||||||
return files;
|
return files;
|
||||||
@@ -61,7 +63,7 @@ export async function publishRelease({ plan, config, api, build, promote, files
|
|||||||
}
|
}
|
||||||
if (!manifest) {
|
if (!manifest) {
|
||||||
const images = await build(plan.version);
|
const images = await build(plan.version);
|
||||||
manifest = manifestFactory(plan, images, config);
|
manifest = await manifestFactory(plan, images, config);
|
||||||
}
|
}
|
||||||
assertManifest(manifest, plan, config);
|
assertManifest(manifest, plan, config);
|
||||||
if (!release) {
|
if (!release) {
|
||||||
|
|||||||
136
scripts/release/tests/ci-permissions.test.mjs
Normal file
136
scripts/release/tests/ci-permissions.test.mjs
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { execFileSync, spawnSync } from 'node:child_process';
|
||||||
|
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
|
||||||
|
const realGit = execFileSync('which', ['git'], { encoding: 'utf8' }).trim();
|
||||||
|
const ciScript = resolve('scripts/release/ci.mjs');
|
||||||
|
|
||||||
|
function fixture(t) {
|
||||||
|
const cwd = mkdtempSync(join(tmpdir(), 'fedeo-ci-permissions-'));
|
||||||
|
t.after(() => rmSync(cwd, { recursive: true, force: true }));
|
||||||
|
const git = args => execFileSync(realGit, args, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim();
|
||||||
|
git(['init', '-b', 'main']);
|
||||||
|
git(['config', 'user.name', 'Release Test']);
|
||||||
|
git(['config', 'user.email', 'release@example.test']);
|
||||||
|
git(['commit', '--allow-empty', '-m', 'chore: baseline']);
|
||||||
|
const baseline = git(['rev-parse', 'HEAD']);
|
||||||
|
const remote = join(cwd, 'remote.git');
|
||||||
|
git(['init', '--bare', remote]);
|
||||||
|
git(['remote', 'add', 'origin', remote]);
|
||||||
|
git(['push', 'origin', 'main']);
|
||||||
|
git(['checkout', '-b', 'dev']);
|
||||||
|
writeFileSync(join(cwd, 'release.config.json'), JSON.stringify({
|
||||||
|
initialVersion: '3.0.0', bootstrapCommit: baseline, branch: 'main',
|
||||||
|
repositoryUrl: 'https://git.example.test/owner/repo', registry: 'registry.example.test/fedeo',
|
||||||
|
images: [{ name: 'backend', context: 'backend', dockerfile: 'backend/Dockerfile' }],
|
||||||
|
}));
|
||||||
|
git(['add', 'release.config.json']);
|
||||||
|
git(['commit', '-m', 'feat: add release configuration']);
|
||||||
|
git(['push', 'origin', 'dev']);
|
||||||
|
const sha = git(['rev-parse', 'HEAD']);
|
||||||
|
const bin = join(cwd, 'bin');
|
||||||
|
mkdirSync(bin);
|
||||||
|
const gitTrace = join(cwd, 'git-calls.jsonl');
|
||||||
|
const dockerTrace = join(cwd, 'docker-calls.jsonl');
|
||||||
|
const registryFile = join(cwd, 'registry.json');
|
||||||
|
writeFileSync(gitTrace, '');
|
||||||
|
writeFileSync(dockerTrace, '');
|
||||||
|
writeFileSync(registryFile, '{}');
|
||||||
|
|
||||||
|
// All Git operations are real and local, except that the simulated CI user cannot push.
|
||||||
|
writeFileSync(join(bin, 'git'), `#!/usr/bin/env node
|
||||||
|
const { appendFileSync } = require('node:fs');
|
||||||
|
const { spawnSync } = require('node:child_process');
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
appendFileSync(process.env.TEST_GIT_TRACE, JSON.stringify(args) + '\\n');
|
||||||
|
if (args[0] === 'push') {
|
||||||
|
process.stderr.write('remote: Forbidden\\nfatal: requested URL returned error: 403\\n');
|
||||||
|
process.exit(128);
|
||||||
|
}
|
||||||
|
if (process.env.TEST_STALE_DEV === 'true' && args[0] === 'ls-remote' && args[2] === 'refs/heads/dev') {
|
||||||
|
process.stdout.write('b'.repeat(40) + '\\trefs/heads/dev\\n');
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
const result = spawnSync(process.env.TEST_REAL_GIT, args, { stdio: 'inherit' });
|
||||||
|
process.exit(result.status ?? 1);
|
||||||
|
`, { mode: 0o755 });
|
||||||
|
|
||||||
|
// Simulate only Docker's registry boundary; no external service or credential is used.
|
||||||
|
writeFileSync(join(bin, 'docker'), `#!/usr/bin/env node
|
||||||
|
const { appendFileSync, readFileSync, writeFileSync } = require('node:fs');
|
||||||
|
const { createHash } = require('node:crypto');
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
appendFileSync(process.env.TEST_DOCKER_TRACE, JSON.stringify(args) + '\\n');
|
||||||
|
const registry = JSON.parse(readFileSync(process.env.TEST_REGISTRY_FILE, 'utf8'));
|
||||||
|
if (args[1] === 'build') {
|
||||||
|
const tag = args[args.indexOf('--tag') + 1];
|
||||||
|
const digest = 'sha256:' + createHash('sha256').update(tag).digest('hex');
|
||||||
|
registry[tag] = digest;
|
||||||
|
registry[tag.slice(0, tag.lastIndexOf(':')) + '@' + digest] = digest;
|
||||||
|
writeFileSync(args[args.indexOf('--metadata-file') + 1], JSON.stringify({ 'containerimage.digest': digest }));
|
||||||
|
} else if (args[1] === 'imagetools' && args[2] === 'create') {
|
||||||
|
const digest = registry[args.at(-1)];
|
||||||
|
if (!digest) process.exit(1);
|
||||||
|
registry[args[args.indexOf('--tag') + 1]] = digest;
|
||||||
|
} else if (args[1] === 'imagetools' && args[2] === 'inspect') {
|
||||||
|
const digest = registry[args[3]];
|
||||||
|
if (!digest) { process.stderr.write('manifest unknown\\n'); process.exit(1); }
|
||||||
|
process.stdout.write(JSON.stringify({ digest }));
|
||||||
|
} else { process.exit(1); }
|
||||||
|
writeFileSync(process.env.TEST_REGISTRY_FILE, JSON.stringify(registry));
|
||||||
|
`, { mode: 0o755 });
|
||||||
|
|
||||||
|
const run = (extra = {}) => spawnSync(process.execPath, [ciScript], {
|
||||||
|
cwd, encoding: 'utf8', timeout: 30_000,
|
||||||
|
env: {
|
||||||
|
...process.env, PATH: `${bin}:${process.env.PATH}`, CI: 'true',
|
||||||
|
FEDEO_REF: 'refs/heads/dev', FEDEO_SHA: sha, FEDEO_EVENT_NAME: 'push', GITEA_TOKEN: '',
|
||||||
|
TEST_REAL_GIT: realGit, TEST_GIT_TRACE: gitTrace, TEST_DOCKER_TRACE: dockerTrace,
|
||||||
|
TEST_REGISTRY_FILE: registryFile, ...extra,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const calls = path => readFileSync(path, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||||
|
return { sha, run, gitCalls: () => calls(gitTrace), dockerCalls: () => calls(dockerTrace), registry: () => JSON.parse(readFileSync(registryFile, 'utf8')) };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('dev builds and retries succeed without a release token or Git push permissions', t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
for (let attempt = 0; attempt < 2; attempt++) {
|
||||||
|
const result = f.run();
|
||||||
|
assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
||||||
|
}
|
||||||
|
assert.ok(f.gitCalls().every(args => !['push', 'commit-tree'].includes(args[0])));
|
||||||
|
const snapshots = Object.keys(f.registry()).filter(tag => tag.includes(`:dev-sha-${f.sha}-`));
|
||||||
|
assert.equal(snapshots.length, 2, 'Reruns must have separate snapshots, even for the same commit');
|
||||||
|
assert.equal(f.registry()['registry.example.test/fedeo/backend:dev'], f.registry()[snapshots[1]]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an outdated dev run keeps its snapshot without advancing the dev channel', t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const result = f.run({ TEST_STALE_DEV: 'true' });
|
||||||
|
assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
||||||
|
assert.equal(f.registry()['registry.example.test/fedeo/backend:dev'], undefined);
|
||||||
|
assert.match(result.stdout, /neuerer dev-Commit/);
|
||||||
|
assert.ok(f.dockerCalls().some(args => args[1] === 'build'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('main still requires a release credential before building or publishing', t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const result = f.run({ FEDEO_REF: 'refs/heads/main' });
|
||||||
|
assert.equal(result.status, 1);
|
||||||
|
assert.match(result.stderr, /RELEASE_TOKEN.*write:repository/);
|
||||||
|
assert.equal(f.dockerCalls().length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a forbidden main lock fails with an actionable message and never builds unlocked', t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const result = f.run({ FEDEO_REF: 'refs/heads/main', GITEA_TOKEN: 'synthetic-test-token' });
|
||||||
|
assert.equal(result.status, 1);
|
||||||
|
assert.match(result.stderr, /main-Release-Sperre benötigt Git-Schreibrechte/);
|
||||||
|
assert.doesNotMatch(result.stderr, /synthetic-test-token/);
|
||||||
|
assert.ok(f.gitCalls().some(args => args[0] === 'push'));
|
||||||
|
assert.equal(f.dockerCalls().length, 0);
|
||||||
|
});
|
||||||
73
scripts/release/tests/migrations.test.mjs
Normal file
73
scripts/release/tests/migrations.test.mjs
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { git, migrationManifest } from '../lib.mjs';
|
||||||
|
|
||||||
|
function fixture(t) {
|
||||||
|
const cwd = mkdtempSync(join(tmpdir(), 'fedeo-migrations-test-'));
|
||||||
|
t.after(() => rmSync(cwd, { recursive: true, force: true }));
|
||||||
|
const run = args => git(args, cwd, { stdio: ['pipe', 'pipe', 'pipe'] });
|
||||||
|
run(['init', '-b', 'main']);
|
||||||
|
run(['config', 'user.name', 'Release Test']);
|
||||||
|
run(['config', 'user.email', 'release@example.test']);
|
||||||
|
// Keep CRLF test bytes in Git regardless of the developer's autocrlf setting.
|
||||||
|
writeFileSync(join(cwd, '.gitattributes'), '*.sql -text\n');
|
||||||
|
const write = (path, data) => {
|
||||||
|
mkdirSync(dirname(join(cwd, path)), { recursive: true });
|
||||||
|
writeFileSync(join(cwd, path), data);
|
||||||
|
};
|
||||||
|
const commit = () => {
|
||||||
|
run(['add', '.']);
|
||||||
|
run(['commit', '--allow-empty', '-m', 'test: add migrations']);
|
||||||
|
return run(['rev-parse', 'HEAD']);
|
||||||
|
};
|
||||||
|
return { cwd, run, write, commit };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('large migration hashes use the complete committed bytes beyond process buffer limits', async t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const path = 'backend/db/migrations/0058_reference_data.sql';
|
||||||
|
f.write(path, '');
|
||||||
|
const expected = createHash('sha256');
|
||||||
|
const chunk = Buffer.alloc(1024 * 1024, Buffer.from(" INSERT INTO reference VALUES ('Grüße');\r\n"));
|
||||||
|
// Exceed both Node's 1 MiB default and the text Git helper's 32 MiB buffer.
|
||||||
|
for (let i = 0; i < 34; i++) {
|
||||||
|
appendFileSync(join(f.cwd, path), chunk);
|
||||||
|
expected.update(chunk);
|
||||||
|
}
|
||||||
|
const sha = f.commit();
|
||||||
|
f.write(path, 'local changes must not affect the release checksum');
|
||||||
|
assert.deepEqual(await migrationManifest(sha, f.cwd), [{ path, sha256: expected.digest('hex') }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('both migration folders are hashed byte-for-byte and non-SQL files are excluded', async t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const files = [
|
||||||
|
['backend/db/migrations/0000_init.sql', Buffer.from(' \r\nSELECT 1;\r\n\t')],
|
||||||
|
['push-server/packages/db/drizzle/0000_init.sql', Buffer.from('SELECT \'äöü\';\n')],
|
||||||
|
];
|
||||||
|
for (const [path, content] of files) f.write(path, content);
|
||||||
|
f.write('backend/db/migrations/meta/snapshot.json', '{}');
|
||||||
|
const sha = f.commit();
|
||||||
|
assert.deepEqual(await migrationManifest(sha, f.cwd), files.map(([path, content]) => ({
|
||||||
|
path, sha256: createHash('sha256').update(content).digest('hex'),
|
||||||
|
})));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a commit without migration folders produces an empty manifest', async t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
assert.deepEqual(await migrationManifest(f.commit(), f.cwd), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an unreadable Git blob rejects the manifest instead of recording a partial checksum', async t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const path = 'backend/db/migrations/0000_missing.sql';
|
||||||
|
f.write(path, 'SELECT 1;');
|
||||||
|
const sha = f.commit();
|
||||||
|
const blob = f.run(['rev-parse', `${sha}:${path}`]);
|
||||||
|
rmSync(join(f.cwd, '.git', 'objects', blob.slice(0, 2), blob.slice(2)));
|
||||||
|
await assert.rejects(async () => migrationManifest(sha, f.cwd), /0000_missing\.sql/);
|
||||||
|
});
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { join, resolve } from 'node:path';
|
import { join, resolve } from 'node:path';
|
||||||
@@ -17,9 +18,9 @@ function fixture(t) {
|
|||||||
run(['commit', '--allow-empty', '-m', 'historic baseline']);
|
run(['commit', '--allow-empty', '-m', 'historic baseline']);
|
||||||
const config = { bootstrapCommit: run(['rev-parse', 'HEAD']), initialVersion: '1.0.0', repositoryUrl: 'https://git.example.test/owner/fedeo' };
|
const config = { bootstrapCommit: run(['rev-parse', 'HEAD']), initialVersion: '1.0.0', repositoryUrl: 'https://git.example.test/owner/fedeo' };
|
||||||
let count = 0;
|
let count = 0;
|
||||||
const commit = message => {
|
const commit = (message, file = 'file') => {
|
||||||
writeFileSync(join(cwd, 'file'), `${++count}\n`);
|
writeFileSync(join(cwd, file), `${++count}\n`);
|
||||||
run(['add', 'file']);
|
run(['add', file]);
|
||||||
run(['commit', '-m', message]);
|
run(['commit', '-m', message]);
|
||||||
return run(['rev-parse', 'HEAD']);
|
return run(['rev-parse', 'HEAD']);
|
||||||
};
|
};
|
||||||
@@ -52,6 +53,51 @@ test('first release uses explicit baseline and excludes legacy history', async t
|
|||||||
assert.doesNotMatch(plan.notes, /historic baseline|compare\/\.\.\./);
|
assert.doesNotMatch(plan.notes, /historic baseline|compare\/\.\.\./);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function divergentHistory(t) {
|
||||||
|
const f = fixture(t);
|
||||||
|
f.run(['branch', 'legacy-main']);
|
||||||
|
f.config.bootstrapCommit = f.commit('chore: versioning baseline');
|
||||||
|
f.commit('feat: current export');
|
||||||
|
f.run(['checkout', 'legacy-main']);
|
||||||
|
f.commit('.gitea/ISSUE_TEMPLATE/bug_report.md aktualisiert', 'legacy-template');
|
||||||
|
const legacyHead = f.commit('feat: historic main feature', 'legacy-feature');
|
||||||
|
f.config.legacyHistoryHeads = [legacyHead];
|
||||||
|
return f;
|
||||||
|
}
|
||||||
|
|
||||||
|
test('the first main merge excludes pinned legacy branch history from validation and notes', async t => {
|
||||||
|
const f = divergentHistory(t);
|
||||||
|
f.run(['checkout', 'main']);
|
||||||
|
f.run(['merge', '--no-ff', 'legacy-main', '-m', 'Merge historic main']);
|
||||||
|
const plan = await f.plan();
|
||||||
|
assert.equal(plan.version, '1.0.0');
|
||||||
|
assert.match(plan.notes, /current export/);
|
||||||
|
assert.doesNotMatch(plan.notes, /historic main feature|ISSUE_TEMPLATE/);
|
||||||
|
writeFileSync(join(f.cwd, 'release.config.json'), JSON.stringify(f.config));
|
||||||
|
const output = execFileSync(process.execPath, [resolve('scripts/release/cli.mjs'), 'check'], { cwd: f.cwd, encoding: 'utf8' });
|
||||||
|
assert.match(output, /1 Commit-Nachrichten geprüft/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('new commits on a legacy branch still affect versions and must use Conventional Commits', async t => {
|
||||||
|
const f = divergentHistory(t);
|
||||||
|
f.run(['checkout', 'main']);
|
||||||
|
f.run(['tag', 'v1.0.0']);
|
||||||
|
f.run(['checkout', 'legacy-main']);
|
||||||
|
f.commit('feat(api)!: replace exported API', 'new-api');
|
||||||
|
f.run(['checkout', 'main']);
|
||||||
|
f.run(['merge', '--no-ff', 'legacy-main', '-m', 'Merge new API']);
|
||||||
|
const plan = await f.plan();
|
||||||
|
assert.equal(plan.version, '2.0.0');
|
||||||
|
assert.match(plan.notes, /replace exported API/);
|
||||||
|
assert.doesNotMatch(plan.notes, /historic main feature/);
|
||||||
|
|
||||||
|
f.run(['checkout', 'legacy-main']);
|
||||||
|
f.commit('Unclassified new change', 'another-change');
|
||||||
|
f.run(['checkout', 'main']);
|
||||||
|
f.run(['merge', '--no-ff', 'legacy-main', '-m', 'Merge another change']);
|
||||||
|
await assert.rejects(f.plan(), /Conventional Commits fehlen:[\s\S]*Unclassified new change/);
|
||||||
|
});
|
||||||
|
|
||||||
test('patch/minor/major releases, multi-digit tags and unrelated component tags', async t => {
|
test('patch/minor/major releases, multi-digit tags and unrelated component tags', async t => {
|
||||||
const f = fixture(t);
|
const f = fixture(t);
|
||||||
f.run(['tag', 'v1.9.0']);
|
f.run(['tag', 'v1.9.0']);
|
||||||
@@ -104,6 +150,14 @@ test('an unknown baseline fails instead of releasing unrelated history', async t
|
|||||||
await assert.rejects(f.plan(), /Ausgangscommit/);
|
await assert.rejects(f.plan(), /Ausgangscommit/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('missing or mutable legacy history heads fail instead of silently skipping validation', async t => {
|
||||||
|
const f = fixture(t);
|
||||||
|
f.config.legacyHistoryHeads = ['f'.repeat(40)];
|
||||||
|
await assert.rejects(f.plan(), /alte Release-Historie .* fehlt/);
|
||||||
|
f.config.legacyHistoryHeads = ['main'];
|
||||||
|
await assert.rejects(f.plan(), /feste vollständige Commit-IDs/);
|
||||||
|
});
|
||||||
|
|
||||||
test('Git lease lock excludes concurrent runners and cannot remove another owner', async t => {
|
test('Git lease lock excludes concurrent runners and cannot remove another owner', async t => {
|
||||||
const f = fixture(t);
|
const f = fixture(t);
|
||||||
const remote = join(f.cwd, 'remote.git');
|
const remote = join(f.cwd, 'remote.git');
|
||||||
|
|||||||
@@ -58,6 +58,26 @@ test('failed build never creates a draft or changes release channels', async ()
|
|||||||
assert.deepEqual(f.events, []);
|
assert.deepEqual(f.events, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('asynchronous manifest generation finishes before creating the release draft', async () => {
|
||||||
|
const f = fixture();
|
||||||
|
f.dependencies.manifestFactory = async () => {
|
||||||
|
await new Promise(resolve => setImmediate(resolve));
|
||||||
|
f.events.push('manifest');
|
||||||
|
return f.manifest;
|
||||||
|
};
|
||||||
|
await publishRelease(f.dependencies);
|
||||||
|
assert.deepEqual(f.events.slice(0, 3), ['build', 'manifest', 'draft']);
|
||||||
|
assert.equal(f.getRelease().draft, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('failed migration hashing never creates a draft or promotes images', async () => {
|
||||||
|
const f = fixture();
|
||||||
|
f.dependencies.manifestFactory = async () => { throw new Error('migration hashing failed'); };
|
||||||
|
await assert.rejects(publishRelease(f.dependencies), /migration hashing failed/);
|
||||||
|
assert.deepEqual(f.events, ['build']);
|
||||||
|
assert.equal(f.getRelease(), null);
|
||||||
|
});
|
||||||
|
|
||||||
test('retry a draft uses saved digests and original notes without rebuilding', async () => {
|
test('retry a draft uses saved digests and original notes without rebuilding', async () => {
|
||||||
const f = fixture();
|
const f = fixture();
|
||||||
f.dependencies.promote = async () => { throw new Error('registry interrupted'); };
|
f.dependencies.promote = async () => { throw new Error('registry interrupted'); };
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ have() {
|
|||||||
|
|
||||||
install_base_packages() {
|
install_base_packages() {
|
||||||
local missing=()
|
local missing=()
|
||||||
for binary in curl openssl; do
|
for binary in curl openssl python3; do
|
||||||
if ! have "$binary"; then
|
if ! have "$binary"; then
|
||||||
missing+=("$binary")
|
missing+=("$binary")
|
||||||
fi
|
fi
|
||||||
@@ -104,7 +104,7 @@ install_base_packages() {
|
|||||||
|
|
||||||
echo "Installiere Basispakete: ${missing[*]}"
|
echo "Installiere Basispakete: ${missing[*]}"
|
||||||
$SUDO apt-get update
|
$SUDO apt-get update
|
||||||
$SUDO apt-get install -y ca-certificates curl openssl
|
$SUDO apt-get install -y ca-certificates curl openssl python3
|
||||||
}
|
}
|
||||||
|
|
||||||
ensure_docker() {
|
ensure_docker() {
|
||||||
@@ -185,6 +185,8 @@ download_selfhost_files() {
|
|||||||
download_file "$FEDEO_RAW_BASE_URL/docker-compose.selfhost.yml" "$FEDEO_DIR/docker-compose.yml"
|
download_file "$FEDEO_RAW_BASE_URL/docker-compose.selfhost.yml" "$FEDEO_DIR/docker-compose.yml"
|
||||||
download_file "$FEDEO_RAW_BASE_URL/.env.example" "$FEDEO_DIR/.env.example"
|
download_file "$FEDEO_RAW_BASE_URL/.env.example" "$FEDEO_DIR/.env.example"
|
||||||
download_file "$FEDEO_RAW_BASE_URL/scripts/selfhost-setup.sh" "$FEDEO_DIR/scripts/selfhost-setup.sh"
|
download_file "$FEDEO_RAW_BASE_URL/scripts/selfhost-setup.sh" "$FEDEO_DIR/scripts/selfhost-setup.sh"
|
||||||
|
download_file "$FEDEO_RAW_BASE_URL/scripts/selfhost-update.py" "$FEDEO_DIR/scripts/selfhost-update.py"
|
||||||
|
download_file "$FEDEO_RAW_BASE_URL/docs/selfhost-updates.md" "$FEDEO_DIR/SELFHOST-UPDATES.md"
|
||||||
chmod +x "$FEDEO_DIR/scripts/selfhost-setup.sh"
|
chmod +x "$FEDEO_DIR/scripts/selfhost-setup.sh"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
631
scripts/selfhost-update.py
Normal file
631
scripts/selfhost-update.py
Normal file
@@ -0,0 +1,631 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""FEDEO host-side update engine. Python 3.10+, Docker Compose 2.24+; no pip packages.
|
||||||
|
|
||||||
|
The CLI and a future host service share Updater. No daemon or HTTP API is installed.
|
||||||
|
All commands operate on an existing local selfhost installation, never a git checkout.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import contextlib
|
||||||
|
import datetime
|
||||||
|
import difflib
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
REPOSITORY = "https://git.federspiel.tech/flfeders/FEDEO"
|
||||||
|
REGISTRY = "git.federspiel.tech/flfeders/fedeo"
|
||||||
|
VERSION = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\Z")
|
||||||
|
FILES = ("release-manifest.json", "docker-compose.selfhost.yml", ".env.example", "selfhost-setup.sh", "CHANGELOG.md")
|
||||||
|
GIB = 1024 ** 3
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def sha(data):
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def atomic(path, data):
|
||||||
|
path = Path(path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
temp = path.with_name(path.name + ".tmp-" + uuid.uuid4().hex)
|
||||||
|
try:
|
||||||
|
with temp.open("xb") as stream:
|
||||||
|
os.chmod(temp, 0o600)
|
||||||
|
stream.write(data if isinstance(data, bytes) else data.encode())
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
os.replace(temp, path)
|
||||||
|
fd = os.open(path.parent, os.O_RDONLY)
|
||||||
|
try:
|
||||||
|
os.fsync(fd)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
finally:
|
||||||
|
temp.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def write_json(path, data):
|
||||||
|
atomic(path, json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def read_json(path, default=None):
|
||||||
|
return json.loads(Path(path).read_text()) if Path(path).exists() else default
|
||||||
|
|
||||||
|
|
||||||
|
def set_version(data, version):
|
||||||
|
# Never source .env as a shell program. Preserve every other byte/setting.
|
||||||
|
value = data.decode()
|
||||||
|
pattern = r"(?m)^(?:export\s+)?FEDEO_VERSION=[^\r\n]*"
|
||||||
|
if re.search(pattern, value):
|
||||||
|
return re.sub(pattern, "FEDEO_VERSION=" + version, value).encode()
|
||||||
|
return (value + ("" if value.endswith("\n") else "\n") + "FEDEO_VERSION=" + version + "\n").encode()
|
||||||
|
|
||||||
|
|
||||||
|
class Https:
|
||||||
|
def __init__(self, token=""):
|
||||||
|
self.token = token
|
||||||
|
|
||||||
|
def get(self, url, limit=8 * 1024 * 1024):
|
||||||
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *args):
|
||||||
|
return None
|
||||||
|
opener = urllib.request.build_opener(NoRedirect)
|
||||||
|
for _ in range(6):
|
||||||
|
parsed = urllib.parse.urlsplit(url)
|
||||||
|
if parsed.scheme != "https" or parsed.username or parsed.password:
|
||||||
|
raise UpdateError("Download benötigt eine HTTPS-Adresse ohne Zugangsdaten.")
|
||||||
|
headers = {"Accept": "application/json", "User-Agent": "FEDEO-Updater/1"}
|
||||||
|
if self.token and parsed.netloc == urllib.parse.urlsplit(REPOSITORY).netloc:
|
||||||
|
headers["Authorization"] = "token " + self.token
|
||||||
|
try:
|
||||||
|
with opener.open(urllib.request.Request(url, headers=headers), timeout=30) as response:
|
||||||
|
data = response.read(limit + 1)
|
||||||
|
if len(data) > limit:
|
||||||
|
raise UpdateError("Release-Datei überschreitet die erlaubte Größe.")
|
||||||
|
return data
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
if error.code in (301, 302, 303, 307, 308) and error.headers.get("Location"):
|
||||||
|
url = urllib.parse.urljoin(url, error.headers["Location"])
|
||||||
|
continue
|
||||||
|
raise UpdateError(f"Download fehlgeschlagen: HTTP {error.code}.") from None
|
||||||
|
except (OSError, urllib.error.URLError):
|
||||||
|
raise UpdateError("HTTPS-Anfrage fehlgeschlagen; Verbindung und Zertifikat prüfen.") from None
|
||||||
|
raise UpdateError("Zu viele Download-Weiterleitungen.")
|
||||||
|
|
||||||
|
def json(self, url):
|
||||||
|
try:
|
||||||
|
return json.loads(self.get(url))
|
||||||
|
except (ValueError, UnicodeError):
|
||||||
|
raise UpdateError("Ungültige JSON-Antwort.") from None
|
||||||
|
|
||||||
|
|
||||||
|
class Releases:
|
||||||
|
def __init__(self, http):
|
||||||
|
self.http = http
|
||||||
|
self.api = REPOSITORY.replace("/flfeders/FEDEO", "/api/v1/repos/flfeders/FEDEO")
|
||||||
|
|
||||||
|
def download(self, requested, destination):
|
||||||
|
if requested:
|
||||||
|
version = requested.removeprefix("v")
|
||||||
|
if not VERSION.fullmatch(version):
|
||||||
|
raise UpdateError("Eine stabile Version wie 3.1.0 angeben.")
|
||||||
|
release = self.http.json(self.api + "/releases/tags/v" + version)
|
||||||
|
else:
|
||||||
|
releases = []
|
||||||
|
for page in range(1, 101):
|
||||||
|
batch = self.http.json(self.api + f"/releases?limit=50&page={page}")
|
||||||
|
releases.extend(r for r in batch if not r.get("draft") and not r.get("prerelease")
|
||||||
|
and VERSION.fullmatch(r.get("tag_name", "").removeprefix("v")))
|
||||||
|
if len(batch) < 50:
|
||||||
|
break
|
||||||
|
if not releases:
|
||||||
|
raise UpdateError("Kein veröffentlichter stabiler Release gefunden.")
|
||||||
|
release = max(releases, key=lambda r: tuple(map(int, r["tag_name"].removeprefix("v").split("."))))
|
||||||
|
version = release["tag_name"].removeprefix("v")
|
||||||
|
if release.get("draft") or release.get("prerelease") or release.get("tag_name") != "v" + version:
|
||||||
|
raise UpdateError("Nur veröffentlichte stabile Releases sind erlaubt.")
|
||||||
|
assets = self.http.json(self.api + f'/releases/{int(release["id"])}/assets')
|
||||||
|
urls = {}
|
||||||
|
for asset in assets:
|
||||||
|
name = asset.get("name")
|
||||||
|
if name in (*FILES, "SHA256SUMS"):
|
||||||
|
url = asset.get("browser_download_url", "")
|
||||||
|
if name in urls or urllib.parse.urlsplit(url).netloc != urllib.parse.urlsplit(REPOSITORY).netloc:
|
||||||
|
raise UpdateError("Doppelte Release-Datei oder unerwartete Download-Herkunft.")
|
||||||
|
urls[name] = url
|
||||||
|
if any(name not in urls for name in (*FILES, "SHA256SUMS")):
|
||||||
|
raise UpdateError("Release enthält nicht alle benötigten Betriebsdateien.")
|
||||||
|
checksum_data = self.http.get(urls["SHA256SUMS"])
|
||||||
|
sums = {}
|
||||||
|
for line in checksum_data.decode().splitlines():
|
||||||
|
match = re.fullmatch(r"([a-f0-9]{64}) ([^/\\]+)", line)
|
||||||
|
if not match or match[2] in sums:
|
||||||
|
raise UpdateError("Ungültige oder doppelte Prüfsummen.")
|
||||||
|
sums[match[2]] = match[1]
|
||||||
|
for name in FILES:
|
||||||
|
data = self.http.get(urls[name])
|
||||||
|
if sha(data) != sums.get(name):
|
||||||
|
raise UpdateError(f"Prüfsumme stimmt nicht: {name}.")
|
||||||
|
atomic(destination / name, data)
|
||||||
|
atomic(destination / "SHA256SUMS", checksum_data)
|
||||||
|
manifest = read_json(destination / "release-manifest.json")
|
||||||
|
tag = self.http.json(self.api + "/tags/v" + version)
|
||||||
|
if (manifest.get("schemaVersion") != 1 or manifest.get("version") != version
|
||||||
|
or manifest.get("tag") != "v" + version or manifest.get("repository") != REPOSITORY
|
||||||
|
or not re.fullmatch(r"[a-f0-9]{40}", manifest.get("commit", ""))
|
||||||
|
or manifest["commit"] != tag.get("commit", {}).get("sha")):
|
||||||
|
raise UpdateError("Release-Manifest, Git-Tag und Version passen nicht zusammen.")
|
||||||
|
for name in ("backend", "frontend"):
|
||||||
|
images = [i for i in manifest.get("images", []) if i.get("name") == name]
|
||||||
|
if (len(images) != 1 or images[0].get("repository") != REGISTRY + "/" + name
|
||||||
|
or not re.fullmatch(r"sha256:[a-f0-9]{64}", images[0].get("digest", ""))):
|
||||||
|
raise UpdateError(f"Ungültiger Image-Digest für {name}.")
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
class Runner:
|
||||||
|
def __init__(self, log):
|
||||||
|
self.log = log
|
||||||
|
# Compose interpolation must use the installation's .env, not host shell settings.
|
||||||
|
allowed = {"PATH", "HOME", "LANG", "LC_ALL", "DOCKER_CONFIG", "DOCKER_HOST", "DOCKER_CONTEXT",
|
||||||
|
"DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "XDG_RUNTIME_DIR", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"}
|
||||||
|
self.env = {k: v for k, v in os.environ.items() if k in allowed}
|
||||||
|
|
||||||
|
def run(self, args, *, output=None, input_file=None, timeout=1800):
|
||||||
|
with contextlib.ExitStack() as stack:
|
||||||
|
log = stack.enter_context(self.log.open("ab"))
|
||||||
|
target = stack.enter_context(Path(output).open("wb")) if output else subprocess.PIPE
|
||||||
|
source = stack.enter_context(Path(input_file).open("rb")) if input_file else subprocess.DEVNULL
|
||||||
|
# Do not log argv/environment: Compose commands may contain secrets.
|
||||||
|
log.write(f"{now()} Start: {args[0]}\n".encode())
|
||||||
|
log.flush()
|
||||||
|
proc = subprocess.Popen(args, stdout=target, stderr=log, stdin=source, env=self.env,
|
||||||
|
start_new_session=True, close_fds=True)
|
||||||
|
try:
|
||||||
|
stdout, _ = proc.communicate(timeout=timeout)
|
||||||
|
except BaseException:
|
||||||
|
with contextlib.suppress(ProcessLookupError):
|
||||||
|
os.killpg(proc.pid, signal.SIGTERM)
|
||||||
|
try:
|
||||||
|
proc.wait(timeout=10)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
with contextlib.suppress(ProcessLookupError):
|
||||||
|
os.killpg(proc.pid, signal.SIGKILL)
|
||||||
|
proc.wait()
|
||||||
|
raise
|
||||||
|
if proc.returncode:
|
||||||
|
if stdout:
|
||||||
|
log.write(stdout)
|
||||||
|
raise UpdateError(f"{args[0]} fehlgeschlagen (Exit {proc.returncode}); geschütztes commands.log prüfen.")
|
||||||
|
return (stdout or b"").decode()
|
||||||
|
|
||||||
|
|
||||||
|
class Updater:
|
||||||
|
def __init__(self, root, *, http=None, runner_factory=Runner):
|
||||||
|
self.root = Path(root).resolve()
|
||||||
|
if not self.root.is_dir() or (self.root / ".git").exists():
|
||||||
|
raise UpdateError("Ein bestehendes Selfhost-Betriebsverzeichnis ohne Git-Checkout angeben.")
|
||||||
|
self.home = self.root / ".updates"
|
||||||
|
self.home.mkdir(mode=0o700, exist_ok=True)
|
||||||
|
if self.home.is_symlink():
|
||||||
|
raise UpdateError(".updates darf kein symbolischer Link sein.")
|
||||||
|
os.chmod(self.home, 0o700)
|
||||||
|
self.http = http or Https(os.environ.get("FEDEO_RELEASE_TOKEN", ""))
|
||||||
|
self.releases = Releases(self.http)
|
||||||
|
self.runner_factory = runner_factory
|
||||||
|
self.state = None
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def lock(self):
|
||||||
|
with (self.home / "lock").open("a") as lock:
|
||||||
|
try:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except BlockingIOError:
|
||||||
|
raise UpdateError("Ein Update oder eine Prüfung läuft bereits.") from None
|
||||||
|
# Children never inherit the lock; the kernel releases it after process death.
|
||||||
|
yield
|
||||||
|
|
||||||
|
def status(self):
|
||||||
|
state = read_json(self.home / "status.json", {"status": "idle"})
|
||||||
|
try:
|
||||||
|
with self.lock():
|
||||||
|
state = read_json(self.home / "status.json", {"status": "idle"})
|
||||||
|
if state.get("status") == "running":
|
||||||
|
state.update(status="interrupted", error="Update-Prozess wurde unterbrochen.", updatedAt=now())
|
||||||
|
write_json(self.home / "status.json", state)
|
||||||
|
write_json(Path(state["job"]) / "status.json", state)
|
||||||
|
except UpdateError:
|
||||||
|
pass
|
||||||
|
return state
|
||||||
|
|
||||||
|
def save(self, phase=None, **values):
|
||||||
|
self.state.update(values, updatedAt=now())
|
||||||
|
if phase:
|
||||||
|
self.state["phase"] = phase
|
||||||
|
write_json(self.home / "status.json", self.state)
|
||||||
|
write_json(self.job / "status.json", self.state)
|
||||||
|
with (self.job / "events.jsonl").open("a") as stream:
|
||||||
|
stream.write(json.dumps({"time": now(), "phase": self.state["phase"], "status": self.state["status"]}) + "\n")
|
||||||
|
|
||||||
|
def compose(self, candidate, *args, **kwargs):
|
||||||
|
prefix = ["docker", "compose", "--project-directory", str(self.root)]
|
||||||
|
if getattr(self, "project", None):
|
||||||
|
prefix += ["--project-name", self.project]
|
||||||
|
prefix += ["--env-file", str(self.job / "target.env" if candidate else self.root / ".env"),
|
||||||
|
"-f", str(self.job / "target.compose.yml" if candidate else self.root / "docker-compose.yml")]
|
||||||
|
if self.override:
|
||||||
|
prefix += ["-f", str(self.override)]
|
||||||
|
return self.runner.run(prefix + list(args), **kwargs)
|
||||||
|
|
||||||
|
def config(self, candidate):
|
||||||
|
return json.loads(self.compose(candidate, "config", "--format", "json"))
|
||||||
|
|
||||||
|
def inspect(self, name):
|
||||||
|
return json.loads(self.runner.run(["docker", "inspect", name]))[0]
|
||||||
|
|
||||||
|
def check_space(self, required, directory):
|
||||||
|
if shutil.disk_usage(directory).free < required:
|
||||||
|
raise UpdateError(f"Zu wenig freier Speicher auf {directory}: mindestens {required // GIB + 1} GiB erforderlich.")
|
||||||
|
|
||||||
|
def prepare(self, version):
|
||||||
|
for name in (".env", "docker-compose.yml"):
|
||||||
|
path = self.root / name
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
raise UpdateError(f"{name} fehlt oder ist ein symbolischer Link.")
|
||||||
|
if any((self.root / name).exists() for name in ("compose.yaml", "compose.yml")):
|
||||||
|
raise UpdateError("Mehrdeutige Compose-Dateien im Betriebsverzeichnis.")
|
||||||
|
overrides = [self.root / name for name in ("docker-compose.override.yml", "docker-compose.override.yaml")
|
||||||
|
if (self.root / name).exists()]
|
||||||
|
if len(overrides) > 1 or any(p.is_symlink() for p in overrides):
|
||||||
|
raise UpdateError("Genau eine reguläre Compose-Override-Datei verwenden.")
|
||||||
|
self.override = overrides[0] if overrides else None
|
||||||
|
self.runner = self.runner_factory(self.job / "commands.log")
|
||||||
|
compose_version = self.runner.run(["docker", "compose", "version", "--short"]).strip().lstrip("v")
|
||||||
|
parts = re.match(r"(\d+)\.(\d+)", compose_version)
|
||||||
|
if not parts or tuple(map(int, parts.groups())) < (2, 24):
|
||||||
|
raise UpdateError("Docker Compose ab Version 2.24 wird benötigt.")
|
||||||
|
info = json.loads(self.runner.run(["docker", "info", "--format", "{{json .}}"] ))
|
||||||
|
if info.get("OSType") != "linux" or info.get("Architecture") not in ("x86_64", "amd64"):
|
||||||
|
raise UpdateError("Diese Releases unterstützen ausschließlich Linux/amd64.")
|
||||||
|
endpoint = self.runner.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"] ).strip()
|
||||||
|
if not os.environ.get("DOCKER_HOST", endpoint).startswith("unix://"):
|
||||||
|
raise UpdateError("Der Updater muss direkt auf dem lokalen Docker-Host laufen.")
|
||||||
|
self.check_space(5 * GIB, info["DockerRootDir"])
|
||||||
|
self.current = self.config(False)
|
||||||
|
self.project = self.current["name"]
|
||||||
|
self.manifest = self.releases.download(version, self.job)
|
||||||
|
self.target = self.manifest["version"]
|
||||||
|
self.save("validating", version=self.target, revision=self.manifest["commit"])
|
||||||
|
old_version = self.current["services"]["backend"].get("environment", {}).get("FEDEO_VERSION", "dev")
|
||||||
|
if VERSION.fullmatch(old_version) and tuple(map(int, old_version.split("."))) >= tuple(map(int, self.target.split("."))):
|
||||||
|
raise UpdateError("Zielversion muss neuer sein; Downgrades und erneute Installation sind gesperrt.")
|
||||||
|
previous = read_json(self.home / "installed.json", {})
|
||||||
|
write_json(self.job / "original-installed.json", previous)
|
||||||
|
previous_manifest = previous.get("manifest", {})
|
||||||
|
old_migrations = {m["path"]: m["sha256"] for m in previous_manifest.get("migrations", [])}
|
||||||
|
new_migrations = {m["path"]: m["sha256"] for m in self.manifest.get("migrations", [])}
|
||||||
|
if any(new_migrations.get(name) != checksum for name, checksum in old_migrations.items()):
|
||||||
|
raise UpdateError("Bereits ausgelieferte Migrationen wurden verändert oder entfernt.")
|
||||||
|
original = (self.root / "docker-compose.yml").read_bytes()
|
||||||
|
self.managed = sha(original) == previous.get("composeSha256")
|
||||||
|
compose = (self.job / "docker-compose.selfhost.yml").read_text()
|
||||||
|
self.images = {}
|
||||||
|
for name in ("backend", "frontend"):
|
||||||
|
image = next(i for i in self.manifest["images"] if i["name"] == name)
|
||||||
|
self.images[name] = image["repository"] + "@" + image["digest"]
|
||||||
|
pattern = r"(?m)^(\s+image:\s*)" + re.escape(image["repository"]) + r":[^\r\n]+$"
|
||||||
|
compose, count = re.subn(pattern, lambda m: m[1] + self.images[name], compose)
|
||||||
|
if count != 1:
|
||||||
|
raise UpdateError(f"Release-Compose enthält kein eindeutiges {name}-Image.")
|
||||||
|
atomic(self.job / "target.compose.yml", compose)
|
||||||
|
atomic(self.job / "target.env", set_version((self.root / ".env").read_bytes(), self.target))
|
||||||
|
diff = "".join(difflib.unified_diff(original.decode().splitlines(True), compose.splitlines(True),
|
||||||
|
fromfile="installed/docker-compose.yml", tofile="release/docker-compose.yml"))
|
||||||
|
atomic(self.job / "compose.diff", diff)
|
||||||
|
self.candidate = self.config(True)
|
||||||
|
self.validate_configs()
|
||||||
|
ids = self.compose(False, "ps", "--all", "--quiet").split()
|
||||||
|
self.containers = [self.inspect(container) for container in ids]
|
||||||
|
write_json(self.job / "original-containers.json", self.containers)
|
||||||
|
if any(c["State"]["Running"] and c["Config"]["Labels"].get("com.docker.compose.oneoff", "False").lower() == "true" for c in self.containers):
|
||||||
|
raise UpdateError("Ein Einmalcontainer läuft noch; dessen Arbeit zuerst abschließen.")
|
||||||
|
self.running = [c["Config"]["Labels"]["com.docker.compose.service"] for c in self.containers
|
||||||
|
if c["State"]["Running"] and c["Config"]["Labels"].get("com.docker.compose.oneoff", "False").lower() != "true"]
|
||||||
|
if not {"backend", "frontend", "db", "minio"}.issubset(self.running):
|
||||||
|
raise UpdateError("Backend, Frontend, Datenbank und MinIO müssen vor dem Update laufen.")
|
||||||
|
if not set(self.running).issubset(self.current["services"]):
|
||||||
|
raise UpdateError("Nicht konfigurierte Container dieses Compose-Projekts laufen noch.")
|
||||||
|
self.before = {str(p): sha(p.read_bytes()) for p in
|
||||||
|
[self.root / ".env", self.root / "docker-compose.yml", *([self.override] if self.override else [])]}
|
||||||
|
database_size = int(self.compose(False, "exec", "-T", "db", "sh", "-ec",
|
||||||
|
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT pg_database_size(current_database())"').strip())
|
||||||
|
minio_size = int(self.runner.run(["du", "-sb", str(self.root / "minio")]).split()[0])
|
||||||
|
self.backup_size = database_size * 2 + minio_size + GIB
|
||||||
|
self.check_space(self.backup_size, self.home)
|
||||||
|
plan = {"version": self.target, "revision": self.manifest["commit"], "previousVersion": old_version,
|
||||||
|
"images": self.images, "composeDiff": str(self.job / "compose.diff"),
|
||||||
|
"releaseNotes": str(self.job / "CHANGELOG.md"), "requiresComposeAdoption": not self.managed,
|
||||||
|
"estimatedBackupBytes": self.backup_size, "services": list(self.candidate["services"])}
|
||||||
|
write_json(self.job / "plan.json", plan)
|
||||||
|
return plan
|
||||||
|
|
||||||
|
def validate_configs(self):
|
||||||
|
old, new = self.current["services"], self.candidate["services"]
|
||||||
|
required = {"backend", "frontend", "db", "minio", "createbuckets", "traefik", "node-exporter"}
|
||||||
|
if set(old) != required or set(new) != required:
|
||||||
|
raise UpdateError("Dieser Updater unterstützt den Standard-Selfhost-Stack mit lokalen Daten und MinIO.")
|
||||||
|
for name in required:
|
||||||
|
if old[name].get("profiles") or new[name].get("profiles") or new[name].get("build"):
|
||||||
|
raise UpdateError("Build-Kontexte und Compose-Profile werden nicht unterstützt.")
|
||||||
|
if old[name].get("volumes", []) != new[name].get("volumes", []):
|
||||||
|
raise UpdateError(f"Datenpfade/Mounts von {name} würden verändert; zuerst manuell migrieren.")
|
||||||
|
if old[name].get("container_name") != new[name].get("container_name"):
|
||||||
|
raise UpdateError(f"Container-Identität von {name} würde verändert.")
|
||||||
|
if self.current.get("networks") != self.candidate.get("networks"):
|
||||||
|
raise UpdateError("Docker-Netzwerke würden verändert.")
|
||||||
|
for name, target, directory in (("db", "/var/lib/postgresql/data", "postgres"), ("minio", "/data", "minio")):
|
||||||
|
volumes = new[name].get("volumes", [])
|
||||||
|
if len(volumes) != 1 or volumes[0].get("type") != "bind" or volumes[0].get("target") != target or volumes[0].get("source") != str(self.root / directory):
|
||||||
|
raise UpdateError(f"{name} benötigt das lokale Standard-Datenverzeichnis.")
|
||||||
|
if (self.root / directory).is_symlink() or not (self.root / directory).is_dir():
|
||||||
|
raise UpdateError(f"Ungültiges Datenverzeichnis: {directory}.")
|
||||||
|
if old[name].get("environment") != new[name].get("environment"):
|
||||||
|
raise UpdateError(f"Zugangsdaten/Konfiguration von {name} würden verändert.")
|
||||||
|
keys = ("POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD") if name == "db" else ("MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD")
|
||||||
|
if any(not new[name].get("environment", {}).get(key) for key in keys):
|
||||||
|
raise UpdateError(f"Pflichteinstellungen von {name} fehlen.")
|
||||||
|
if old["db"]["image"] != new["db"]["image"]:
|
||||||
|
raise UpdateError("Ein Wechsel der PostgreSQL-Version benötigt eine separate Datenbankmigration.")
|
||||||
|
for name in ("backend", "frontend"):
|
||||||
|
healthcheck = new[name].get("healthcheck", {})
|
||||||
|
if new[name]["image"] != self.images[name] or healthcheck.get("disable") or not healthcheck.get("test") or healthcheck["test"] == ["NONE"]:
|
||||||
|
raise UpdateError(f"{name}: Release-Digest oder Healthcheck wurde überschrieben.")
|
||||||
|
for name in ("minio", "createbuckets"):
|
||||||
|
if not re.search(r"@sha256:[a-f0-9]{64}$", new[name]["image"]):
|
||||||
|
raise UpdateError("MinIO und mc müssen auf feste Digests gepinnt sein.")
|
||||||
|
env = new["backend"].get("environment", {})
|
||||||
|
for key in ("DATABASE_URL", "S3_ENDPOINT", "S3_BUCKET", "S3_ACCESS_KEY", "S3_SECRET_KEY", "ENCRYPTION_KEY", "COOKIE_SECRET", "JWT_SECRET"):
|
||||||
|
if not env.get(key) or env[key] != old["backend"].get("environment", {}).get(key):
|
||||||
|
raise UpdateError(f"Backend-Einstellung {key} fehlt oder würde verändert.")
|
||||||
|
if (env.get("FEDEO_FILE_BACKEND", "s3") != "s3" or old["backend"].get("environment", {}).get("FEDEO_FILE_BACKEND", "s3") != "s3" or urllib.parse.urlsplit(env["S3_ENDPOINT"]).hostname != "minio"
|
||||||
|
or urllib.parse.urlsplit(env["DATABASE_URL"]).hostname != "db"):
|
||||||
|
raise UpdateError("Externe Datenbanken/Dateispeicher benötigen einen eigenen Backup-Adapter.")
|
||||||
|
database = urllib.parse.urlsplit(env["DATABASE_URL"])
|
||||||
|
db_env = new["db"]["environment"]
|
||||||
|
if (urllib.parse.unquote(database.path.lstrip("/")) != db_env["POSTGRES_DB"]
|
||||||
|
or urllib.parse.unquote(database.username or "") != db_env["POSTGRES_USER"]
|
||||||
|
or urllib.parse.unquote(database.password or "") != db_env["POSTGRES_PASSWORD"]
|
||||||
|
or database.port not in (None, 5432)):
|
||||||
|
raise UpdateError("DATABASE_URL muss auf die gesicherte lokale PostgreSQL-Datenbank zeigen.")
|
||||||
|
url = new["frontend"].get("environment", {}).get("NUXT_PUBLIC_API_BASE", "")
|
||||||
|
parts = urllib.parse.urlsplit(url)
|
||||||
|
if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.path.rstrip("/") != "/backend" or parts.query or parts.fragment:
|
||||||
|
raise UpdateError("Die öffentliche API-Adresse muss https://DOMAIN/backend entsprechen.")
|
||||||
|
self.public = urllib.parse.urlunsplit((parts.scheme, parts.netloc, "", "", ""))
|
||||||
|
for name, service in new.items():
|
||||||
|
for key, value in service.get("environment", {}).items():
|
||||||
|
if value is None or (value == "" and key in ("DOMAIN", "POSTGRES_PASSWORD", "COOKIE_SECRET", "JWT_SECRET", "ENCRYPTION_KEY")):
|
||||||
|
raise UpdateError(f"Fehlende Umgebungsvariable: {name}.{key}.")
|
||||||
|
|
||||||
|
def backup(self):
|
||||||
|
directory = self.job / "backup"
|
||||||
|
directory.mkdir(mode=0o700)
|
||||||
|
for name in (".env", "docker-compose.yml", ".env.example", "scripts/selfhost-setup.sh"):
|
||||||
|
if (self.root / name).is_file():
|
||||||
|
atomic(directory / "config" / name, (self.root / name).read_bytes())
|
||||||
|
if self.override:
|
||||||
|
atomic(directory / "config" / self.override.name, self.override.read_bytes())
|
||||||
|
write_json(directory / "containers.json", self.containers)
|
||||||
|
self.compose(False, "exec", "-T", "db", "sh", "-ec",
|
||||||
|
'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom', output=directory / "database.dump")
|
||||||
|
self.compose(False, "exec", "-T", "db", "pg_restore", "--list", input_file=directory / "database.dump")
|
||||||
|
self.compose(False, "exec", "-T", "db", "sh", "-ec",
|
||||||
|
'exec pg_dumpall -U "$POSTGRES_USER" --globals-only', output=directory / "globals.sql")
|
||||||
|
paths = ["minio"]
|
||||||
|
if (self.root / "traefik/letsencrypt").is_dir():
|
||||||
|
paths.append("traefik/letsencrypt")
|
||||||
|
self.runner.run(["tar", "-cpf", str(directory / "files.tar"), "-C", str(self.root), *paths], timeout=7200)
|
||||||
|
self.runner.run(["tar", "-tf", str(directory / "files.tar")], output=directory / "files.list", timeout=7200)
|
||||||
|
checksums = {}
|
||||||
|
for path in sorted(directory.rglob("*")):
|
||||||
|
if path.is_file():
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
checksums[str(path.relative_to(directory))] = digest.hexdigest()
|
||||||
|
write_json(directory / "complete.json", {"createdAt": now(), "database": "database.dump", "files": "files.tar", "sha256": checksums})
|
||||||
|
self.save("backed_up", backup=str(directory))
|
||||||
|
|
||||||
|
def verify(self, version, revision=None):
|
||||||
|
# Traefik may need a moment to discover newly created healthy containers.
|
||||||
|
for attempt in range(6):
|
||||||
|
try:
|
||||||
|
for path in ("/backend/health", "/api/health"):
|
||||||
|
value = self.http.json(self.public + path)
|
||||||
|
if value.get("status") != "ok" or value.get("product") != "FEDEO" or value.get("version") != version or (revision and value.get("revision") != revision):
|
||||||
|
raise UpdateError("Öffentliche Funktionsprüfung meldet eine falsche Version oder einen Fehler.")
|
||||||
|
return
|
||||||
|
except UpdateError:
|
||||||
|
if attempt == 5:
|
||||||
|
raise
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
def apply(self, accept_compose):
|
||||||
|
if not self.managed and not accept_compose:
|
||||||
|
raise UpdateError("Compose-Datei ist noch nicht verwaltet oder wurde bearbeitet. compose.diff prüfen, Anpassungen in die Override-Datei übertragen und --accept-compose-replacement verwenden.")
|
||||||
|
self.save("pulling")
|
||||||
|
self.compose(True, "pull", timeout=3600)
|
||||||
|
self.check_space(self.backup_size, self.home)
|
||||||
|
if any(sha(Path(path).read_bytes()) != value for path, value in self.before.items()):
|
||||||
|
raise UpdateError("Konfiguration wurde während der Vorbereitung verändert; erneut prüfen.")
|
||||||
|
self.save("maintenance", recoveryRequired=True, maintenance=True, runningServices=self.running)
|
||||||
|
# Stop every writer and public entry point. The database stays up for pg_dump.
|
||||||
|
self.compose(False, "stop", "--timeout", "60", *[s for s in self.running if s != "db"])
|
||||||
|
self.backup()
|
||||||
|
if any(sha(Path(path).read_bytes()) != value for path, value in self.before.items()):
|
||||||
|
raise UpdateError("Konfiguration wurde während des Backups verändert; erneut prüfen.")
|
||||||
|
# From here recovery must be explicit: migrations or infrastructure can alter persisted data.
|
||||||
|
self.save("applying", mutationStarted=True)
|
||||||
|
atomic(self.root / "docker-compose.yml", (self.job / "target.compose.yml").read_bytes())
|
||||||
|
atomic(self.root / ".env", (self.job / "target.env").read_bytes())
|
||||||
|
atomic(self.root / ".env.example", (self.job / ".env.example").read_bytes())
|
||||||
|
atomic(self.root / "scripts/selfhost-setup.sh", (self.job / "selfhost-setup.sh").read_bytes())
|
||||||
|
os.chmod(self.root / "scripts/selfhost-setup.sh", 0o700)
|
||||||
|
self.compose(True, "up", "-d", "--no-deps", "--wait", "--wait-timeout", "180", "db", "minio", "node-exporter")
|
||||||
|
self.compose(True, "run", "--rm", "--no-deps", "createbuckets")
|
||||||
|
self.save("migrating", migrationContainer="fedeo-update-" + self.job.name)
|
||||||
|
self.compose(True, "run", "--rm", "--no-deps", "--name", self.state["migrationContainer"],
|
||||||
|
"-e", "FEDEO_RUN_MIGRATIONS=false", "--entrypoint", "npm", "backend", "run", "migrate")
|
||||||
|
self.save("starting")
|
||||||
|
self.compose(True, "up", "-d", "--wait", "--wait-timeout", "600", timeout=900)
|
||||||
|
self.save("verifying")
|
||||||
|
for name, image in self.images.items():
|
||||||
|
container_id = self.compose(True, "ps", "--quiet", name).strip()
|
||||||
|
container = self.inspect(container_id)
|
||||||
|
image_id = self.runner.run(["docker", "image", "inspect", image, "--format", "{{.Id}}"] ).strip()
|
||||||
|
if container["Image"] != image_id or container.get("State", {}).get("Health", {}).get("Status") != "healthy":
|
||||||
|
raise UpdateError(f"{name} verwendet nicht das erwartete gesunde Image.")
|
||||||
|
self.verify(self.target, self.manifest["commit"])
|
||||||
|
write_json(self.home / "installed.json", {"manifest": self.manifest, "composeSha256": sha((self.root / "docker-compose.yml").read_bytes()), "installedAt": now()})
|
||||||
|
self.save("complete", status="succeeded", recoveryRequired=False, maintenance=False)
|
||||||
|
|
||||||
|
def execute(self, command, version=None, accept_compose=False):
|
||||||
|
with self.lock():
|
||||||
|
previous = read_json(self.home / "status.json", {})
|
||||||
|
if previous.get("recoveryRequired"):
|
||||||
|
raise UpdateError("Ein vorheriges Update benötigt Wiederherstellung. status und docs/selfhost-updates.md prüfen; danach resolve ausführen.")
|
||||||
|
self.job = self.home / (datetime.datetime.now().strftime("%Y%m%d-%H%M%S-") + uuid.uuid4().hex[:8])
|
||||||
|
self.job.mkdir(mode=0o700)
|
||||||
|
self.state = {"status": "running", "phase": "preparing", "job": str(self.job), "command": command,
|
||||||
|
"startedAt": now(), "recoveryRequired": False, "mutationStarted": False}
|
||||||
|
self.save()
|
||||||
|
try:
|
||||||
|
plan = self.prepare(version)
|
||||||
|
if command == "check":
|
||||||
|
self.save("checked", status="checked")
|
||||||
|
else:
|
||||||
|
self.apply(accept_compose)
|
||||||
|
return {**self.state, "plan": plan}
|
||||||
|
except BaseException as error:
|
||||||
|
if self.state.get("maintenance") and not self.state.get("mutationStarted"):
|
||||||
|
try:
|
||||||
|
self.compose(False, "start", *self.running)
|
||||||
|
self.save(recoveryRequired=False, maintenance=False)
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
elif self.state.get("mutationStarted"):
|
||||||
|
try:
|
||||||
|
self.compose(True, "stop", "--timeout", "60", "backend", "frontend", "traefik")
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
self.save(status="interrupted" if isinstance(error, (KeyboardInterrupt, SystemExit)) else "failed",
|
||||||
|
error=str(error) if isinstance(error, UpdateError) else type(error).__name__)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def resolve(self, version):
|
||||||
|
if not version or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.+-]{0,100}", version):
|
||||||
|
raise UpdateError("Die manuell wiederhergestellte Version mit --version angeben.")
|
||||||
|
with self.lock():
|
||||||
|
state = read_json(self.home / "status.json", {})
|
||||||
|
if not state.get("recoveryRequired"):
|
||||||
|
raise UpdateError("Kein offener Wiederherstellungsfall.")
|
||||||
|
self.job = Path(state["job"])
|
||||||
|
self.state = state
|
||||||
|
self.runner = self.runner_factory(self.job / "commands.log")
|
||||||
|
self.override = next((self.root / name for name in ("docker-compose.override.yml", "docker-compose.override.yaml") if (self.root / name).exists()), None)
|
||||||
|
current = self.config(False)
|
||||||
|
self.public = current["services"]["frontend"]["environment"]["NUXT_PUBLIC_API_BASE"].removesuffix("/backend")
|
||||||
|
if state.get("migrationContainer"):
|
||||||
|
remaining = self.runner.run(["docker", "ps", "--all", "--quiet", "--filter", "name=^/" + state["migrationContainer"] + "$"]).strip()
|
||||||
|
if remaining:
|
||||||
|
raise UpdateError("Den verbliebenen Migrationscontainer zuerst prüfen und manuell entfernen.")
|
||||||
|
# Older releases have /version and /health, but no frontend health endpoint.
|
||||||
|
version_info = self.http.json(self.public + "/backend/version")
|
||||||
|
if version_info.get("product") != "FEDEO" or version_info.get("version") != version:
|
||||||
|
raise UpdateError("Die wiederhergestellte Backend-Version stimmt nicht.")
|
||||||
|
if self.http.json(self.public + "/backend/health").get("status") != "ok":
|
||||||
|
raise UpdateError("Das wiederhergestellte Backend ist nicht erreichbar.")
|
||||||
|
self.http.get(self.public + "/")
|
||||||
|
original_images = {c["Config"]["Labels"]["com.docker.compose.service"]: c["Image"]
|
||||||
|
for c in read_json(self.job / "original-containers.json", [])}
|
||||||
|
manifest = read_json(self.job / "release-manifest.json")
|
||||||
|
for name in ("backend", "frontend", "db", "minio"):
|
||||||
|
container_id = self.compose(False, "ps", "--quiet", name).strip()
|
||||||
|
container = self.inspect(container_id) if container_id else {}
|
||||||
|
container_state = container.get("State", {})
|
||||||
|
if not container_state.get("Running") or container_state.get("Health", {}).get("Status") not in (None, "healthy"):
|
||||||
|
raise UpdateError(f"{name} läuft noch nicht.")
|
||||||
|
if name in ("backend", "frontend"):
|
||||||
|
if version == manifest["version"]:
|
||||||
|
image = next(i for i in manifest["images"] if i["name"] == name)
|
||||||
|
expected = self.runner.run(["docker", "image", "inspect", image["repository"] + "@" + image["digest"], "--format", "{{.Id}}"] ).strip()
|
||||||
|
else:
|
||||||
|
expected = original_images.get(name)
|
||||||
|
if not expected or container.get("Image") != expected:
|
||||||
|
raise UpdateError(f"{name} entspricht weder dem gewählten Zielrelease noch dem gesicherten Originalstand.")
|
||||||
|
if version == manifest["version"]:
|
||||||
|
self.verify(version, manifest["commit"])
|
||||||
|
write_json(self.home / "installed.json", {"manifest": manifest,
|
||||||
|
"composeSha256": sha((self.job / "target.compose.yml").read_bytes()), "installedAt": now()})
|
||||||
|
else:
|
||||||
|
previous = read_json(self.job / "original-installed.json", {})
|
||||||
|
if previous:
|
||||||
|
write_json(self.home / "installed.json", previous)
|
||||||
|
else:
|
||||||
|
(self.home / "installed.json").unlink(missing_ok=True)
|
||||||
|
self.save(status="recovered", phase="resolved", recoveryRequired=False, maintenance=False, recoveredVersion=version)
|
||||||
|
return self.state
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
if sys.version_info < (3, 10):
|
||||||
|
print("Python ab Version 3.10 wird benötigt.", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("command", choices=("check", "apply", "status", "resolve"))
|
||||||
|
parser.add_argument("--directory", default=os.environ.get("FEDEO_DIR", "/opt/fedeo"))
|
||||||
|
parser.add_argument("--version", help="Zielversion, z. B. 3.1.0; check wählt sonst den neuesten stabilen Release")
|
||||||
|
parser.add_argument("--accept-compose-replacement", action="store_true", help="Zuvor geprüften Austausch einer nicht verwalteten Compose-Datei akzeptieren")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
if args.command == "apply" and not args.version:
|
||||||
|
parser.error("apply benötigt eine explizite --version; zuerst check ausführen")
|
||||||
|
os.umask(0o077)
|
||||||
|
signal.signal(signal.SIGTERM, lambda *_: sys.exit(143))
|
||||||
|
try:
|
||||||
|
updater = Updater(args.directory)
|
||||||
|
if args.command == "status":
|
||||||
|
result = updater.status()
|
||||||
|
elif args.command == "resolve":
|
||||||
|
result = updater.resolve(args.version)
|
||||||
|
else:
|
||||||
|
result = updater.execute(args.command, args.version, args.accept_compose_replacement)
|
||||||
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
|
return 0
|
||||||
|
except (UpdateError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as error:
|
||||||
|
print(str(error) if isinstance(error, UpdateError) else f"Update abgebrochen: {type(error).__name__}; geschütztes Statusprotokoll prüfen.", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
406
scripts/update/tests/test_updater.py
Normal file
406
scripts/update/tests/test_updater.py
Normal file
@@ -0,0 +1,406 @@
|
|||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import io
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
spec = importlib.util.spec_from_file_location('updater', ROOT / 'scripts/selfhost-update.py')
|
||||||
|
u = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(u)
|
||||||
|
|
||||||
|
|
||||||
|
def data(value):
|
||||||
|
return json.dumps(value).encode()
|
||||||
|
|
||||||
|
|
||||||
|
class Http:
|
||||||
|
def __init__(self):
|
||||||
|
self.manifest = {'schemaVersion': 1, 'version': '3.1.0', 'tag': 'v3.1.0', 'commit': 'a' * 40,
|
||||||
|
'repository': u.REPOSITORY, 'migrations': [],
|
||||||
|
'images': [{'name': name, 'repository': u.REGISTRY + '/' + name,
|
||||||
|
'digest': 'sha256:' + char * 64} for name, char in [('backend', 'a'), ('frontend', 'b')]]}
|
||||||
|
self.files = {name: (ROOT / {'docker-compose.selfhost.yml': 'docker-compose.selfhost.yml',
|
||||||
|
'.env.example': '.env.example', 'selfhost-setup.sh': 'scripts/selfhost-setup.sh'}[name]).read_bytes()
|
||||||
|
for name in ['docker-compose.selfhost.yml', '.env.example', 'selfhost-setup.sh']}
|
||||||
|
self.files.update({'release-manifest.json': data(self.manifest), 'CHANGELOG.md': b'New release'})
|
||||||
|
self.refresh()
|
||||||
|
self.requests = []
|
||||||
|
self.bad_health = False
|
||||||
|
self.release = {'id': 1, 'tag_name': 'v3.1.0', 'draft': False, 'prerelease': False}
|
||||||
|
self.fail_download = False
|
||||||
|
|
||||||
|
def refresh(self):
|
||||||
|
self.files['SHA256SUMS'] = ''.join(hashlib.sha256(blob).hexdigest() + ' ' + name + '\n'
|
||||||
|
for name, blob in self.files.items() if name != 'SHA256SUMS').encode()
|
||||||
|
|
||||||
|
def get(self, url, **kwargs):
|
||||||
|
self.requests.append(url)
|
||||||
|
if url.endswith('/'):
|
||||||
|
return b'<!doctype html><html>FEDEO</html>'
|
||||||
|
if self.fail_download:
|
||||||
|
raise u.UpdateError('Download fehlgeschlagen')
|
||||||
|
return self.files[url.rsplit('/', 1)[-1]]
|
||||||
|
|
||||||
|
def json(self, url):
|
||||||
|
self.requests.append(url)
|
||||||
|
if url.endswith('/releases/tags/v3.1.0'):
|
||||||
|
return self.release
|
||||||
|
if '/releases?limit=' in url:
|
||||||
|
return [self.release, {'tag_name': 'v9.0.0', 'draft': True}, {'tag_name': 'v4.0.0-rc.1', 'prerelease': True}]
|
||||||
|
if url.endswith('/releases/1/assets'):
|
||||||
|
return [{'name': name, 'browser_download_url': u.REPOSITORY + '/assets/' + name} for name in self.files]
|
||||||
|
if url.endswith('/tags/v3.1.0'):
|
||||||
|
return {'commit': {'sha': 'a' * 40}}
|
||||||
|
if url.endswith('/backend/version'):
|
||||||
|
return {'product': 'FEDEO', 'version': '3.1.0'}
|
||||||
|
if url.endswith('/health'):
|
||||||
|
return {'status': 'ok', 'product': 'FEDEO', 'version': '3.0.0' if self.bad_health else '3.1.0', 'revision': 'a' * 40}
|
||||||
|
raise AssertionError(url)
|
||||||
|
|
||||||
|
|
||||||
|
class Runner:
|
||||||
|
"""Real Compose parsing with a simulated daemon; no containers are started."""
|
||||||
|
def __init__(self, log, http):
|
||||||
|
self.log = log
|
||||||
|
self.http = http
|
||||||
|
self.calls = []
|
||||||
|
self.fail = None
|
||||||
|
self.on_pull = None
|
||||||
|
self.interrupt = None
|
||||||
|
|
||||||
|
def run(self, args, output=None, input_file=None, **kwargs):
|
||||||
|
self.calls.append(args)
|
||||||
|
if self.interrupt and self.interrupt(args):
|
||||||
|
raise KeyboardInterrupt()
|
||||||
|
if self.fail and self.fail(args):
|
||||||
|
raise u.UpdateError('simulated command failure')
|
||||||
|
if args[:3] == ['docker', 'compose', 'version']:
|
||||||
|
return '2.39.1'
|
||||||
|
if args[:2] == ['docker', 'info']:
|
||||||
|
return json.dumps({'OSType': 'linux', 'Architecture': 'amd64', 'DockerRootDir': '/tmp'})
|
||||||
|
if args[:3] == ['docker', 'context', 'inspect']:
|
||||||
|
return 'unix:///var/run/docker.sock'
|
||||||
|
if args[:3] == ['docker', 'image', 'inspect']:
|
||||||
|
return 'image-' + args[3].rsplit('/', 1)[-1].split('@')[0]
|
||||||
|
if args[:2] == ['docker', 'inspect']:
|
||||||
|
name = args[2]
|
||||||
|
return json.dumps([{'Image': 'image-' + name, 'Config': {'Labels': {'com.docker.compose.service': name}},
|
||||||
|
'State': {'Running': name != 'createbuckets', 'Health': {'Status': 'healthy'}}}])
|
||||||
|
if 'config' in args:
|
||||||
|
result = subprocess.run(args, env={'PATH': os.environ['PATH'], 'HOME': os.environ['HOME']},
|
||||||
|
capture_output=True, text=True)
|
||||||
|
if result.returncode:
|
||||||
|
raise u.UpdateError('Compose invalid: ' + result.stderr)
|
||||||
|
return result.stdout
|
||||||
|
if 'ps' in args:
|
||||||
|
if args[:2] == ['docker', 'ps']:
|
||||||
|
return ''
|
||||||
|
if args[-1] in ('backend', 'frontend', 'db', 'minio'):
|
||||||
|
return args[-1]
|
||||||
|
return 'backend\nfrontend\ndb\nminio\ntraefik\nnode-exporter\ncreatebuckets\n'
|
||||||
|
if 'pull' in args and self.on_pull:
|
||||||
|
self.on_pull()
|
||||||
|
if any('pg_database_size' in part for part in args):
|
||||||
|
return '1000'
|
||||||
|
if args[0] == 'du':
|
||||||
|
return '1000\tminio'
|
||||||
|
if output:
|
||||||
|
Path(output).write_bytes(b'PGDMP fake dump')
|
||||||
|
if args[0] == 'tar' and '-cpf' in args:
|
||||||
|
Path(args[2]).write_bytes(b'backup')
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
(self.root / 'minio').mkdir()
|
||||||
|
(self.root / 'postgres').mkdir()
|
||||||
|
(self.root / '.env').write_bytes(u.set_version((ROOT / '.env.example').read_bytes(), '3.0.0'))
|
||||||
|
(self.root / 'docker-compose.yml').write_bytes((ROOT / 'docker-compose.selfhost.yml').read_bytes())
|
||||||
|
self.original_env = (self.root / '.env').read_bytes()
|
||||||
|
self.original_compose = (self.root / 'docker-compose.yml').read_bytes()
|
||||||
|
self.http = Http()
|
||||||
|
self.runner = Runner(self.root / 'log', self.http)
|
||||||
|
self.updater = u.Updater(self.root, http=self.http, runner_factory=lambda log: self.runner)
|
||||||
|
self.updater.check_space = lambda *_: None
|
||||||
|
self.env_patch = patch.dict(os.environ, {'DOCKER_HOST': 'unix:///var/run/docker.sock'})
|
||||||
|
self.env_patch.start()
|
||||||
|
self.sleep_patch = patch.object(u.time, 'sleep')
|
||||||
|
self.sleep_patch.start()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.env_patch.stop()
|
||||||
|
self.sleep_patch.stop()
|
||||||
|
self.temp.cleanup()
|
||||||
|
|
||||||
|
def run_update(self, **kwargs):
|
||||||
|
return self.updater.execute('apply', '3.1.0', accept_compose=True, **kwargs)
|
||||||
|
|
||||||
|
def called(self, value):
|
||||||
|
return any(value in args for args in self.runner.calls)
|
||||||
|
|
||||||
|
def assert_untouched(self):
|
||||||
|
self.assertEqual((self.root / '.env').read_bytes(), self.original_env)
|
||||||
|
self.assertEqual((self.root / 'docker-compose.yml').read_bytes(), self.original_compose)
|
||||||
|
self.assertFalse(self.called('stop'))
|
||||||
|
|
||||||
|
def test_success_pins_images_preserves_configuration_and_checks_both_public_services(self):
|
||||||
|
result = self.run_update()
|
||||||
|
self.assertEqual(result['status'], 'succeeded')
|
||||||
|
self.assertFalse(result['recoveryRequired'])
|
||||||
|
self.assertEqual((self.root / '.env').read_bytes(), u.set_version(self.original_env, '3.1.0'))
|
||||||
|
self.assertIn(b'backend@sha256:' + b'a' * 64, (self.root / 'docker-compose.yml').read_bytes())
|
||||||
|
backup = Path(result['backup'])
|
||||||
|
self.assertTrue((backup / 'complete.json').exists())
|
||||||
|
self.assertEqual((backup / 'config/.env').read_bytes(), self.original_env)
|
||||||
|
self.assertTrue(any(url.endswith('/backend/health') for url in self.http.requests))
|
||||||
|
self.assertTrue(any(url.endswith('/api/health') for url in self.http.requests))
|
||||||
|
self.assertEqual((self.root / '.env').stat().st_mode & 0o777, 0o600)
|
||||||
|
self.assertEqual((self.root / '.updates').stat().st_mode & 0o777, 0o700)
|
||||||
|
self.assertFalse(self.called('down'))
|
||||||
|
self.assertFalse(self.called('--remove-orphans'))
|
||||||
|
calls = self.runner.calls
|
||||||
|
self.assertLess(next(i for i, a in enumerate(calls) if 'pull' in a), next(i for i, a in enumerate(calls) if 'stop' in a))
|
||||||
|
self.assertLess(next(i for i, a in enumerate(calls) if 'pg_restore' in a), next(i for i, a in enumerate(calls) if 'migrate' in a))
|
||||||
|
|
||||||
|
def test_check_only_creates_a_plan_and_ignores_unpublished_releases(self):
|
||||||
|
result = self.updater.execute('check')
|
||||||
|
self.assertEqual(result['version'], '3.1.0')
|
||||||
|
self.assertEqual(result['status'], 'checked')
|
||||||
|
self.assertTrue(Path(result['plan']['composeDiff']).exists())
|
||||||
|
self.assert_untouched()
|
||||||
|
self.assertFalse(self.called('pull'))
|
||||||
|
|
||||||
|
def test_checksum_failure_does_not_stop_anything(self):
|
||||||
|
self.http.files['docker-compose.selfhost.yml'] += b'\n# tampered'
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Prüfsumme'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_download_failure_does_not_stop_anything(self):
|
||||||
|
self.http.fail_download = True
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Download'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_draft_is_rejected_even_when_explicitly_selected(self):
|
||||||
|
self.http.release['draft'] = True
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'veröffentlichte'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_manifest_commit_must_match_git_tag(self):
|
||||||
|
self.http.manifest['commit'] = 'b' * 40
|
||||||
|
self.http.files['release-manifest.json'] = data(self.http.manifest)
|
||||||
|
self.http.refresh()
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Git-Tag'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_invalid_compose_fails_before_stopping(self):
|
||||||
|
self.http.files['docker-compose.selfhost.yml'] += b'\nbroken: [\n'
|
||||||
|
self.http.refresh()
|
||||||
|
with self.assertRaises(u.UpdateError):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_unknown_base_requires_review(self):
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'compose.diff'):
|
||||||
|
self.updater.execute('apply', '3.1.0')
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_override_is_preserved_but_cannot_replace_release_images(self):
|
||||||
|
override = self.root / 'docker-compose.override.yml'
|
||||||
|
contents = 'services:\n frontend:\n environment:\n MY_CUSTOM_SETTING: preserved\n'
|
||||||
|
override.write_text(contents)
|
||||||
|
self.run_update()
|
||||||
|
self.assertEqual(override.read_text(), contents)
|
||||||
|
self.assertEqual(json.loads((self.root / '.updates/installed.json').read_text())['manifest']['version'], '3.1.0')
|
||||||
|
|
||||||
|
def test_override_cannot_disable_healthcheck(self):
|
||||||
|
(self.root / 'docker-compose.override.yml').write_text('services:\n backend:\n healthcheck:\n disable: true\n')
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Healthcheck'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_changed_mount_is_rejected(self):
|
||||||
|
self.http.files['docker-compose.selfhost.yml'] = self.http.files['docker-compose.selfhost.yml'].replace(b'./postgres:', b'./different:')
|
||||||
|
self.http.refresh()
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Datenpfade'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_external_storage_fails_before_maintenance(self):
|
||||||
|
(self.root / '.env').write_bytes(self.original_env.replace(b'http://minio:9000', b'https://storage.example.test'))
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Backup-Adapter'):
|
||||||
|
self.run_update()
|
||||||
|
self.assertFalse(self.called('stop'))
|
||||||
|
|
||||||
|
def test_database_url_must_match_the_database_being_backed_up(self):
|
||||||
|
(self.root / '.env').write_bytes(self.original_env.replace(b'@db:5432/fedeo', b'@db:5432/other_database'))
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'gesicherte lokale'):
|
||||||
|
self.run_update()
|
||||||
|
self.assertFalse(self.called('stop'))
|
||||||
|
|
||||||
|
def test_missing_required_secret_is_rejected(self):
|
||||||
|
(self.root / '.env').write_bytes(self.original_env.replace(b'ENCRYPTION_KEY=', b'IGNORED_KEY='))
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'ENCRYPTION_KEY'):
|
||||||
|
self.run_update()
|
||||||
|
self.assertFalse(self.called('stop'))
|
||||||
|
|
||||||
|
def test_pull_failure_never_stops_old_stack(self):
|
||||||
|
self.runner.fail = lambda args: 'pull' in args
|
||||||
|
with self.assertRaises(u.UpdateError):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_configuration_changed_during_pull_is_detected(self):
|
||||||
|
self.runner.on_pull = lambda: (self.root / '.env').write_bytes(self.original_env + b'\nNEW=value\n')
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'während'):
|
||||||
|
self.run_update()
|
||||||
|
self.assertFalse(self.called('stop'))
|
||||||
|
|
||||||
|
def test_backup_failure_restarts_previous_services_and_preserves_configuration(self):
|
||||||
|
self.runner.fail = lambda args: any('pg_dump ' in arg for arg in args)
|
||||||
|
with self.assertRaises(u.UpdateError):
|
||||||
|
self.run_update()
|
||||||
|
self.assertTrue(self.called('start'))
|
||||||
|
self.assertFalse(self.called('migrate'))
|
||||||
|
self.assertFalse(self.updater.status()['recoveryRequired'])
|
||||||
|
self.assertEqual((self.root / '.env').read_bytes(), self.original_env)
|
||||||
|
self.assertEqual((self.root / 'docker-compose.yml').read_bytes(), self.original_compose)
|
||||||
|
|
||||||
|
def test_migration_failure_keeps_recovery_state_and_never_starts_old_application(self):
|
||||||
|
self.runner.fail = lambda args: 'migrate' in args
|
||||||
|
with self.assertRaises(u.UpdateError):
|
||||||
|
self.run_update()
|
||||||
|
state = self.updater.status()
|
||||||
|
self.assertEqual(state['phase'], 'migrating')
|
||||||
|
self.assertTrue(state['recoveryRequired'])
|
||||||
|
self.assertTrue(Path(state['backup']).exists())
|
||||||
|
self.assertFalse(self.called('start'))
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Wiederherstellung'):
|
||||||
|
self.run_update()
|
||||||
|
|
||||||
|
def test_interrupt_during_migration_is_persisted(self):
|
||||||
|
self.runner.interrupt = lambda args: 'migrate' in args
|
||||||
|
with self.assertRaises(KeyboardInterrupt):
|
||||||
|
self.run_update()
|
||||||
|
self.assertEqual(self.updater.status()['status'], 'interrupted')
|
||||||
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
||||||
|
self.assertFalse(self.called('start'))
|
||||||
|
|
||||||
|
def test_process_death_is_detected_and_does_not_erase_recovery_state(self):
|
||||||
|
job = self.root / '.updates/interrupted'
|
||||||
|
job.mkdir()
|
||||||
|
u.write_json(self.root / '.updates/status.json', {'status': 'running', 'job': str(job), 'recoveryRequired': True})
|
||||||
|
self.assertEqual(self.updater.status()['status'], 'interrupted')
|
||||||
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Wiederherstellung'):
|
||||||
|
self.run_update()
|
||||||
|
|
||||||
|
def test_lock_prevents_parallel_updates(self):
|
||||||
|
with self.updater.lock():
|
||||||
|
other = u.Updater(self.root, http=self.http)
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'läuft bereits'):
|
||||||
|
other.execute('check')
|
||||||
|
|
||||||
|
def test_old_migrations_must_not_change(self):
|
||||||
|
u.write_json(self.root / '.updates/installed.json', {'manifest': {'migrations': [{'path': 'old.sql', 'sha256': 'a' * 64}]}})
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Migrationen'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_insufficient_space_does_not_stop_services(self):
|
||||||
|
self.updater.check_space = lambda *_: (_ for _ in ()).throw(u.UpdateError('Zu wenig freier Speicher'))
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Speicher'):
|
||||||
|
self.run_update()
|
||||||
|
self.assert_untouched()
|
||||||
|
|
||||||
|
def test_failed_health_verification_does_not_mark_release_installed(self):
|
||||||
|
self.http.bad_health = True
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Funktionsprüfung'):
|
||||||
|
self.run_update()
|
||||||
|
self.assertEqual(self.updater.status()['phase'], 'verifying')
|
||||||
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
||||||
|
self.assertFalse((self.root / '.updates/installed.json').exists())
|
||||||
|
self.assertEqual(self.runner.calls[-1][-3:], ['backend', 'frontend', 'traefik'])
|
||||||
|
|
||||||
|
def test_resolve_requires_a_working_manually_recovered_installation(self):
|
||||||
|
self.runner.fail = lambda args: 'migrate' in args
|
||||||
|
with self.assertRaises(u.UpdateError):
|
||||||
|
self.run_update()
|
||||||
|
self.runner.fail = None
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Version stimmt nicht'):
|
||||||
|
self.updater.resolve('3.0.0')
|
||||||
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
||||||
|
self.assertEqual(self.updater.resolve('3.1.0')['status'], 'recovered')
|
||||||
|
self.assertFalse(self.updater.status()['recoveryRequired'])
|
||||||
|
self.assertEqual(u.read_json(self.root / '.updates/installed.json')['manifest']['version'], '3.1.0')
|
||||||
|
|
||||||
|
def test_second_managed_update_does_not_need_adoption(self):
|
||||||
|
u.write_json(self.root / '.updates/installed.json', {'composeSha256': u.sha(self.original_compose)})
|
||||||
|
self.assertEqual(self.updater.execute('apply', '3.1.0')['status'], 'succeeded')
|
||||||
|
|
||||||
|
def test_downgrade_is_rejected(self):
|
||||||
|
(self.root / '.env').write_bytes(u.set_version(self.original_env, '3.2.0'))
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Downgrades'):
|
||||||
|
self.run_update()
|
||||||
|
self.assertFalse(self.called('stop'))
|
||||||
|
|
||||||
|
def test_backup_integrity_manifest_is_complete(self):
|
||||||
|
result = self.run_update()
|
||||||
|
backup = Path(result['backup'])
|
||||||
|
manifest = u.read_json(backup / 'complete.json')
|
||||||
|
for name in ('database.dump', 'globals.sql', 'files.tar', 'config/.env', 'config/docker-compose.yml'):
|
||||||
|
self.assertEqual(u.sha((backup / name).read_bytes()), manifest['sha256'][name])
|
||||||
|
|
||||||
|
|
||||||
|
class HttpsTests(unittest.TestCase):
|
||||||
|
def test_token_is_removed_after_cross_origin_redirect(self):
|
||||||
|
requests = []
|
||||||
|
|
||||||
|
class Opener:
|
||||||
|
def open(self, request, timeout):
|
||||||
|
requests.append(request)
|
||||||
|
if len(requests) == 1:
|
||||||
|
raise u.urllib.error.HTTPError(request.full_url, 302, 'redirect',
|
||||||
|
{'Location': 'https://objects.example.test/file'}, None)
|
||||||
|
return io.BytesIO(b'asset')
|
||||||
|
|
||||||
|
with patch.object(u.urllib.request, 'build_opener', return_value=Opener()):
|
||||||
|
self.assertEqual(u.Https('private-token').get(u.REPOSITORY + '/asset'), b'asset')
|
||||||
|
self.assertEqual(requests[0].get_header('Authorization'), 'token private-token')
|
||||||
|
self.assertIsNone(requests[1].get_header('Authorization'))
|
||||||
|
|
||||||
|
def test_insecure_redirect_is_rejected(self):
|
||||||
|
class Opener:
|
||||||
|
def open(self, request, timeout):
|
||||||
|
raise u.urllib.error.HTTPError(request.full_url, 302, 'redirect',
|
||||||
|
{'Location': 'http://objects.example.test/file'}, None)
|
||||||
|
|
||||||
|
with patch.object(u.urllib.request, 'build_opener', return_value=Opener()):
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'HTTPS'):
|
||||||
|
u.Https().get(u.REPOSITORY + '/asset')
|
||||||
|
|
||||||
|
def test_download_size_is_limited(self):
|
||||||
|
class Opener:
|
||||||
|
def open(self, request, timeout):
|
||||||
|
return io.BytesIO(b'123456')
|
||||||
|
|
||||||
|
with patch.object(u.urllib.request, 'build_opener', return_value=Opener()):
|
||||||
|
with self.assertRaisesRegex(u.UpdateError, 'Größe'):
|
||||||
|
u.Https().get(u.REPOSITORY + '/asset', limit=5)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user