fix(storage): preserve explicit S3 MIME types in SeaweedFS
This commit is contained in:
@@ -13,7 +13,8 @@ Die fachliche Abnahme erfolgt zusätzlich auf der konkreten Instanz.
|
||||
## Was der Pilot leistet und wo seine Grenze liegt
|
||||
|
||||
- Vorhandenes MinIO verwenden; kein Abruf von `minio/minio` oder `mc` erforderlich.
|
||||
- Feste SeaweedFS-Version 4.47; Offline-Images, SHA-256-geprüftes Paket.
|
||||
- SeaweedFS 4.47 mit dokumentierter FEDEO-Metadatenkorrektur; Offline-Images,
|
||||
SHA-256-geprüftes Paket.
|
||||
- Alle Objekte des einzigen FEDEO-Buckets übernehmen, inklusive Exporte und
|
||||
Objekte ohne aktuellen Datenbankverweis. Schlüssel und Metadaten bleiben erhalten.
|
||||
- Alle bekannten Schreiber anhalten, ihre automatischen Neustarts sperren,
|
||||
@@ -40,6 +41,27 @@ auf SeaweedFS und der Standardinstaller sind nachfolgende Arbeitspakete.
|
||||
Wer diese Einschränkungen noch nicht im Betrieb tragen möchte, führt zunächst
|
||||
nur die Vorprüfung durch und bleibt mit der Instanz auf MinIO.
|
||||
|
||||
### Warum das Pilotimage eine gezielte Korrektur enthält
|
||||
|
||||
Das unveränderte SeaweedFS 4.47 entfernt beim Speichern den expliziten MIME-Typ
|
||||
`application/octet-stream`. Bei einem `.pdf`-Schlüssel liefern HEAD und GET
|
||||
dadurch später `application/pdf`. Diese Abweichung wurde beim ersten betreuten
|
||||
Versuch durch den strengen Metadatenvergleich erkannt; die Instanz wurde vor
|
||||
Schreibfreigabe erfolgreich auf MinIO zurückgeführt.
|
||||
|
||||
`scripts/storage/Dockerfile.seaweedfs` baut deshalb die feste Herstellerrevision
|
||||
`c5073360007d28385a33426a42ac3e4ec504c5a3` mit
|
||||
`preserve-s3-content-type.patch` als `4.47-fedeo.1`. Die Korrektur bewahrt den
|
||||
expliziten Typ für S3-Einträge; die Normalisierung gewöhnlicher Dateisystemeinträge
|
||||
bleibt erhalten. Quellarchiv, Go-Buildimage und Hersteller-Basisimage sind per
|
||||
SHA-256 festgelegt. Gezielte Go-Tests laufen beim Build; das Paket enthält den
|
||||
Buildplan und Patch mit Prüfsummen. Der S3-Vertrag prüft den generischen MIME-Typ
|
||||
zusätzlich mit HEAD, GET und dem Original-Backend-SDK.
|
||||
|
||||
Dies ist eine eigene Buildvariante. Bei einem späteren SeaweedFS-Update muss die
|
||||
Korrektur weitergeführt oder durch einen nachgewiesenen Herstellerfix ersetzt
|
||||
werden; das ungeänderte Herstellerimage ist für diese 1:1-Übernahme nicht freigegeben.
|
||||
|
||||
## Voraussetzungen, die der Plan prüft
|
||||
|
||||
Linux amd64, Python ab 3.10, lokaler Docker-Daemon, Docker Compose und `tar`/`du`.
|
||||
@@ -211,7 +233,7 @@ nicht automatisch gelöscht.
|
||||
## Nachgewiesene Tests und ausstehende Freigaben
|
||||
|
||||
Der lokale S3-Test verwendet das offizielle MinIO-Release-Binary vom 7. September
|
||||
2025 und SeaweedFS 4.47 mit festem Digest. 1.005 Objekte / 1.073.752.750 Bytes,
|
||||
2025 und SeaweedFS 4.47 mit der dokumentierten MIME-Korrektur. 1.005 Objekte / 1.073.752.750 Bytes,
|
||||
einschließlich 1-GiB-Grenzdatei, Pagination, Sonderzeichen, Metadaten, leeren
|
||||
Dateien, Streaming, 24-Stunden-Signaturen, fremdem Bucket und ungültigen Zugängen
|
||||
werden geprüft. Beschädigte Zielobjekte werden erkannt und beim Fortsetzen
|
||||
|
||||
18
scripts/storage/Dockerfile.seaweedfs
Normal file
18
scripts/storage/Dockerfile.seaweedfs
Normal file
@@ -0,0 +1,18 @@
|
||||
FROM golang:1.26-alpine@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c AS builder
|
||||
RUN apk add --no-cache patch
|
||||
WORKDIR /src
|
||||
ADD --checksum=sha256:8b3823d4ca2df8250d7e6f35774228cc4ff01ffe2f8c10656df074a8db2f48a5 https://codeload.github.com/seaweedfs/seaweedfs/tar.gz/c50733600 /tmp/source.tar.gz
|
||||
RUN tar -xzf /tmp/source.tar.gz --strip-components=1 && rm /tmp/source.tar.gz
|
||||
COPY preserve-s3-content-type.patch /tmp/preserve-s3-content-type.patch
|
||||
RUN patch -p1 < /tmp/preserve-s3-content-type.patch
|
||||
ENV CGO_ENABLED=0
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
|
||||
go test -p 4 ./weed/filer -run '^TestFilerStoreWrapperMimeNormalization$' -count=1 -v \
|
||||
&& go build -p 4 -trimpath -ldflags '-s -w -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=c5073360007d28385a33426a42ac3e4ec504c5a3-fedeo.1' -o /out/weed ./weed
|
||||
|
||||
FROM chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
|
||||
COPY --from=builder /out/weed /usr/bin/weed
|
||||
COPY --from=builder /src/LICENSE /usr/share/doc/fedeo-seaweedfs/LICENSE
|
||||
COPY preserve-s3-content-type.patch /usr/share/doc/fedeo-seaweedfs/preserve-s3-content-type.patch
|
||||
LABEL io.fedeo.storage.patch="preserve-s3-content-type-v1" \
|
||||
org.opencontainers.image.revision="c5073360007d28385a33426a42ac3e4ec504c5a3-fedeo.1"
|
||||
@@ -10,6 +10,7 @@ Lokaler Test (Linux amd64, Docker, Node 22):
|
||||
npm ci --prefix scripts/storage
|
||||
npm test --prefix scripts/storage
|
||||
docker pull chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
|
||||
docker build -f scripts/storage/Dockerfile.seaweedfs -t fedeo-seaweedfs:4.47-fedeo.1 scripts/storage
|
||||
docker build -f scripts/storage/tests/Dockerfile.minio -t fedeo-minio-fixture:2025-09-07 scripts/storage/tests
|
||||
docker build -t fedeo-storage-tools:pilot scripts/storage
|
||||
node scripts/storage/tests/integration.mjs
|
||||
@@ -37,7 +38,16 @@ SHA-256-Checksum-Header. Die Pilot-Anwendung erhält
|
||||
`AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED`; der Vertragstest prüft diese
|
||||
Einstellung mit leeren und nicht leeren Streams. SigV4 bleibt aktiv.
|
||||
|
||||
SeaweedFS-Herkunft: [Release 4.47](https://github.com/seaweedfs/seaweedfs/releases/tag/4.47).
|
||||
Das Pilotimage `4.47-fedeo.1` korrigiert SeaweedFS' Normalisierung expliziter
|
||||
S3-MIME-Typen. `application/octet-stream` muss auch für `.pdf` erhalten bleiben.
|
||||
Der Dockerfile pinnt Herstellerquelle, Buildimage und Laufzeitbasis; der beigefügte
|
||||
Patch umfasst die Korrektur und Go-Tests für alle drei Schreibpfade. Der S3-Test
|
||||
prüft HEAD/GET, Kopie und Restore einschließlich dieses Falls. Das ungeänderte
|
||||
Herstellerimage wird vom neuen Vertragstest gezielt abgewiesen. Bei Folgeupdates
|
||||
die Korrektur beibehalten oder deren Übernahme durch den Hersteller nachweisen.
|
||||
|
||||
SeaweedFS-Herkunft: [Release 4.47](https://github.com/seaweedfs/seaweedfs/releases/tag/4.47),
|
||||
[betroffene Normalisierung](https://github.com/seaweedfs/seaweedfs/blob/c5073360007d28385a33426a42ac3e4ec504c5a3/weed/filer/filerstore_wrapper.go).
|
||||
Das isolierte MinIO-Testimage verwendet das offizielle, per SHA-256 verifizierte
|
||||
[Release-Binary](https://github.com/minio/minio/releases/tag/RELEASE.2025-09-07T16-13-09Z),
|
||||
da die früheren Containerreferenzen nicht mehr abrufbar sind. Es wird nicht im
|
||||
|
||||
@@ -5,7 +5,9 @@ storage_output="${1:-$storage_repo/.release/storage-pilot}"
|
||||
test ! -e "$storage_output"
|
||||
mkdir -p "$storage_output/scripts/storage" "$storage_output/docs"
|
||||
storage_weed="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["image"])' "$storage_repo/scripts/storage/seaweedfs.json")"
|
||||
if ! docker image inspect "$storage_weed" >/dev/null 2>&1; then docker pull "$storage_weed"; fi
|
||||
if ! docker image inspect "$storage_weed" >/dev/null 2>&1; then
|
||||
docker build -f "$storage_repo/scripts/storage/Dockerfile.seaweedfs" -t "$storage_weed" "$storage_repo/scripts/storage"
|
||||
fi
|
||||
docker build -t fedeo-storage-tools:pilot "$storage_repo/scripts/storage"
|
||||
storage_tools_id="$(docker image inspect fedeo-storage-tools:pilot --format '{{.Id}}')"
|
||||
storage_weed_id="$(docker image inspect "$storage_weed" --format '{{.Id}}')"
|
||||
@@ -14,7 +16,7 @@ storage_weed_local="fedeo-seaweedfs:pilot-${storage_weed_id#sha256:}"
|
||||
docker image tag "$storage_tools_id" "$storage_tools"
|
||||
docker image tag "$storage_weed_id" "$storage_weed_local"
|
||||
cp "$storage_repo/scripts/selfhost-storage-migrate.py" "$storage_repo/scripts/selfhost-update.py" "$storage_output/scripts/"
|
||||
cp "$storage_repo/scripts/storage/"{s3.mjs,cli.mjs,seaweedfs.json,start-seaweedfs.sh,package.json,package-lock.json} "$storage_output/scripts/storage/"
|
||||
cp "$storage_repo/scripts/storage/"{s3.mjs,cli.mjs,seaweedfs.json,start-seaweedfs.sh,package.json,package-lock.json,Dockerfile.seaweedfs,preserve-s3-content-type.patch} "$storage_output/scripts/storage/"
|
||||
cp "$storage_repo/scripts/storage/pilot.py" "$storage_output/pilot.py"
|
||||
cp "$storage_repo/docs/minio-umstellung-pilot.md" "$storage_output/docs/"
|
||||
docker image save -o "$storage_output/images.tar" "$storage_tools" "$storage_weed_local"
|
||||
@@ -31,8 +33,11 @@ for p in sorted(root.rglob('*')):
|
||||
with p.open('rb') as f:
|
||||
for chunk in iter(lambda:f.read(1024*1024), b''): h.update(chunk)
|
||||
checksums[str(p.relative_to(root))]=h.hexdigest()
|
||||
upstream=next(line.split()[1] for line in (root/'scripts/storage/Dockerfile.seaweedfs').read_text().splitlines()
|
||||
if line.startswith('FROM chrislusf/seaweedfs:'))
|
||||
(root/'bundle.json').write_text(json.dumps({'schemaVersion':1,'toolsImage':sys.argv[2],
|
||||
'seaweedImage':sys.argv[3],'upstreamSeaweedImage':sys.argv[4],'sha256':checksums},indent=2)+'\n')
|
||||
'seaweedImage':sys.argv[3],'upstreamSeaweedImage':upstream,'seaweedBuild':'4.47-fedeo.1',
|
||||
'sha256':checksums},indent=2)+'\n')
|
||||
PY
|
||||
tar -czf "$storage_output.tar.gz" -C "$(dirname "$storage_output")" "$(basename "$storage_output")"
|
||||
(cd "$(dirname "$storage_output")" && sha256sum "$(basename "$storage_output").tar.gz" > "$(basename "$storage_output").tar.gz.sha256")
|
||||
|
||||
70
scripts/storage/preserve-s3-content-type.patch
Normal file
70
scripts/storage/preserve-s3-content-type.patch
Normal file
@@ -0,0 +1,70 @@
|
||||
--- a/weed/filer/filerstore_wrapper.go
|
||||
+++ b/weed/filer/filerstore_wrapper.go
|
||||
@@ -213,7 +213,8 @@
|
||||
if entry == nil {
|
||||
return
|
||||
}
|
||||
- if !entry.IsDirectory() && entry.Mime == "application/octet-stream" {
|
||||
+ // An explicit S3 Content-Type is object metadata, not a filesystem default.
|
||||
+ if !entry.IsDirectory() && !entry.isS3Entry() && entry.Mime == "application/octet-stream" {
|
||||
entry.Mime = ""
|
||||
}
|
||||
}
|
||||
--- a/weed/filer/filerstore_wrapper_test.go
|
||||
+++ b/weed/filer/filerstore_wrapper_test.go
|
||||
@@ -8,6 +8,7 @@
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -17,6 +18,7 @@
|
||||
name string
|
||||
mode os.FileMode
|
||||
wantMime string
|
||||
+ extended map[string][]byte
|
||||
}{
|
||||
{
|
||||
name: "files strip octet-stream",
|
||||
@@ -24,6 +26,18 @@
|
||||
wantMime: "",
|
||||
},
|
||||
{
|
||||
+ name: "S3 owner keeps explicit octet-stream",
|
||||
+ mode: 0o660,
|
||||
+ wantMime: "application/octet-stream",
|
||||
+ extended: map[string][]byte{s3_constants.ExtAmzOwnerKey: []byte("test-owner")},
|
||||
+ },
|
||||
+ {
|
||||
+ name: "S3 ETag keeps explicit octet-stream",
|
||||
+ mode: 0o660,
|
||||
+ wantMime: "application/octet-stream",
|
||||
+ extended: map[string][]byte{s3_constants.ExtETagKey: []byte("test-etag")},
|
||||
+ },
|
||||
+ {
|
||||
name: "directories keep octet-stream",
|
||||
mode: os.ModeDir | 0o770,
|
||||
wantMime: "application/octet-stream",
|
||||
@@ -38,6 +52,12 @@
|
||||
name: "insert",
|
||||
run: func(fsw *FilerStoreWrapper, ctx context.Context, entry *Entry) error {
|
||||
return fsw.InsertEntry(ctx, entry)
|
||||
+ },
|
||||
+ },
|
||||
+ {
|
||||
+ name: "insert known absent",
|
||||
+ run: func(fsw *FilerStoreWrapper, ctx context.Context, entry *Entry) error {
|
||||
+ return fsw.InsertEntryKnownAbsent(ctx, entry)
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -55,6 +75,7 @@
|
||||
wrapper := NewFilerStoreWrapper(store)
|
||||
entry := &Entry{
|
||||
FullPath: util.FullPath("/buckets/test/object"),
|
||||
+ Extended: tt.extended,
|
||||
Attr: Attr{
|
||||
Mode: tt.mode,
|
||||
Mime: "application/octet-stream",
|
||||
@@ -250,6 +250,14 @@ export async function contract(s3, config, Bucket, { foreignBucket, probePath }
|
||||
requireThat(head.ContentLength === body.length && head.Metadata.test === 'fedeo', 'HEAD-Vertrag fehlgeschlagen.');
|
||||
const got = await call(s3, 'GetObject', { Bucket, Key });
|
||||
requireThat(Buffer.from(await got.Body.transformToByteArray()).equals(body), 'GET-Vertrag fehlgeschlagen.');
|
||||
// Generic MIME types are explicit S3 metadata even for a .pdf key. The
|
||||
// upstream filer used to drop octet-stream and infer PDF on later reads.
|
||||
await call(s3, 'PutObject', { Bucket, Key, Body: body, ContentType: 'application/octet-stream', Metadata: { test: 'fedeo' } });
|
||||
const genericHead = await call(s3, 'HeadObject', { Bucket, Key });
|
||||
const genericGet = await call(s3, 'GetObject', { Bucket, Key });
|
||||
const genericBody = Buffer.from(await genericGet.Body.transformToByteArray());
|
||||
requireThat(genericHead.ContentType === 'application/octet-stream' && genericGet.ContentType === 'application/octet-stream'
|
||||
&& genericHead.Metadata.test === 'fedeo' && genericBody.equals(body), 'Expliziter S3-MIME-Typ wurde verändert.');
|
||||
// Application exports use streams. Exercise exactly the SDK environment setting
|
||||
// applied by the pilot Compose file, including empty streams.
|
||||
for (const bytes of [Buffer.alloc(0), Buffer.alloc(128 * 1024, 42)]) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"image": "chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882",
|
||||
"image": "fedeo-seaweedfs:4.47-fedeo.1",
|
||||
"entrypoint": ["/bin/sh", "/etc/fedeo/start-seaweedfs.sh"],
|
||||
"command": [
|
||||
"server", "-dir=/data", "-master.dir=/data/master", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
|
||||
|
||||
@@ -10,6 +10,7 @@ import { client, call, contract, inventory, transfer, verifyManifest, MAX_OBJECT
|
||||
|
||||
const docker = (...args) => execFileSync('docker', args, { encoding: 'utf8', timeout: 120_000, stdio: ['ignore', 'pipe', 'pipe'] }).trim();
|
||||
const spec = JSON.parse(await readFile(new URL('../seaweedfs.json', import.meta.url), 'utf8'));
|
||||
if (process.env.FEDEO_SEAWEED_TEST_IMAGE) spec.image = process.env.FEDEO_SEAWEED_TEST_IMAGE;
|
||||
const root = await mkdtemp(join(tmpdir(), 'fedeo-storage-contract-'));
|
||||
const prefix = `fedeo-contract-${randomUUID().slice(0, 8)}`;
|
||||
const containers = [];
|
||||
@@ -84,7 +85,7 @@ try {
|
||||
`const s=require('node:net').connect(${port},${JSON.stringify(ip)});s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));s.setTimeout(1500,()=>process.exit(1))`));
|
||||
}
|
||||
for (let i = 0; i < 1003; i++) await call(source, 'PutObject', { Bucket, Key: `tenant/filesbyid/${i}/ä + % #.txt`,
|
||||
Body: Buffer.from(`fixture-${i}`), ContentType: 'text/plain', Metadata: { custom: 'preserved' } });
|
||||
Body: Buffer.from(`fixture-${i}`), ContentType: i === 0 ? 'application/octet-stream' : 'text/plain', Metadata: { custom: 'preserved' } });
|
||||
await call(source, 'PutObject', { Bucket, Key: 'empty', Body: Buffer.alloc(0), ContentType: 'application/octet-stream' });
|
||||
await call(source, 'PutObject', { Bucket, Key: 'large.zip', ContentLength: MAX_OBJECT_BYTES,
|
||||
Body: Readable.from((function* () { for (let i = 0; i < MAX_OBJECT_BYTES / (1024 * 1024); i++) yield Buffer.alloc(1024 * 1024, i % 256); })()),
|
||||
|
||||
Reference in New Issue
Block a user