feat(updater): support updates after verified SeaweedFS migration

This commit is contained in:
2026-09-28 15:47:06 +02:00
parent 790f20b499
commit 2b947c5e8f
20 changed files with 1074 additions and 67 deletions

View File

@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Real migration -> adoption -> two updates, including a backup restore, on synthetic data.
Requires local storage test images and registry:2; never contacts a FEDEO instance.
Release HTTP/health responses are fixtures; Compose, registry, SDK, PostgreSQL,
backup, restore and provider validation use real processes and containers.
"""
import copy
import importlib.util
import json
from pathlib import Path
import subprocess
import tarfile
import tempfile
ROOT = Path(__file__).resolve().parents[3]
spec = importlib.util.spec_from_file_location('storage_fixture', ROOT / 'scripts/storage/tests/host_integration.py')
fixture = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fixture)
m, docker = fixture.m, fixture.docker
u = m.u
class Http:
version = '3.4.0'
def json(self, url):
return {'status': 'ok', 'product': 'FEDEO', 'version': self.version, 'revision': 'a' * 40}
class Releases:
def __init__(self, original, images):
self.original, self.images = original, images
def download(self, version, destination):
config = copy.deepcopy(self.original)
for name, image in self.images.items():
config['services'][name]['image'] = image['repository'] + ':fixture'
config['services']['frontend']['entrypoint'] = ['node']
config['services']['frontend']['command'] = ['-e', 'setInterval(()=>{},1000)']
for name in ('traefik', 'node-exporter'):
config['services'][name]['image'] = json.loads(docker('image', 'inspect', 'node:22-alpine'))[0]['Id']
config['services']['backend']['environment']['FEDEO_VERSION'] = version
path = destination / 'fixture.json'
u.write_json(path, m.literal(config))
rendered = docker('compose', '-f', path, 'config', '--format', 'yaml')
(destination / 'docker-compose.selfhost.yml').write_text(rendered)
for name in ('.env.example', 'selfhost-setup.sh', 'CHANGELOG.md'):
(destination / name).write_text('# Synthetic update fixture\n')
manifest = {'schemaVersion': 1, 'version': version, 'commit': 'a' * 40,
'images': list(self.images.values()), 'migrations': []}
u.write_json(destination / 'release-manifest.json', manifest)
return manifest
def run():
root, project = fixture.fixture()
registry = project + '-registry'
registry_id = None
work = Path(tempfile.mkdtemp(prefix='fedeo-storage-update-images-'))
try:
config = m.resolved(u.read_json(root / 'docker-compose.yml'))
config['services']['backend']['environment'].update(ENCRYPTION_KEY='fixture-key', COOKIE_SECRET='fixture-$-cookie', JWT_SECRET='fixture-jwt')
config['services']['frontend']['environment'] = {'NUXT_PUBLIC_API_BASE': 'https://fixture.example.test/backend'}
u.write_json(root / 'docker-compose.yml', m.literal(config))
docker('compose', '-p', project, '-f', root / 'docker-compose.yml', 'up', '-d', '--pull', 'never', '--wait', '--wait-timeout', '90')
fixture.seed(root, project)
# Include the stopped Matrix orphan seen on the first real instance.
(root / 'matrix/postgres').mkdir(parents=True)
(root / 'matrix/postgres/retained.txt').write_text('retired Matrix')
retained = docker('create', '--pull', 'never', '--restart=no', '--network', project + '-internal',
'--label', 'com.docker.compose.project=' + project,
'--label', 'com.docker.compose.service=matrix-db',
'-v', str(root / 'matrix/postgres') + ':/var/lib/postgresql/data', 'node:22-alpine')
migration = m.Migration(root, 'fedeo-storage-tools:pilot')
with migration.engine.lock():
migration.plan()
migration.migrate(True)
migration.activate()
print(json.dumps({'phase': 'migration_complete', 'evidence': str(root)}), flush=True)
for name in u.CORE_SERVICES:
migration.wait_ready(docker('compose', '-p', project, '-f', root / 'docker-compose.yml', 'ps', '-q', name))
http = Http()
updater = u.Updater(root, http=http)
assert updater.adopt_storage()['status'] == 'adopted'
print(json.dumps({'phase': 'storage_adopted'}), flush=True)
registry_id = docker('run', '-d', '--pull', 'never', '--name', registry, '-p', '127.0.0.1::5000', 'registry:2')
port = json.loads(docker('inspect', registry_id))[0]['NetworkSettings']['Ports']['5000/tcp'][0]['HostPort']
repository = 'localhost:' + port + '/fedeo-update-fixture'
(work / 'Dockerfile').write_text('FROM fedeo-storage-tools:pilot\n'
'RUN node -e "const fs=require(\'fs\');const p=JSON.parse(fs.readFileSync(\'package.json\'));'
'p.scripts={migrate:\'node --version\'};fs.writeFileSync(\'package.json\',JSON.stringify(p))"\n'
'ENTRYPOINT ["node"]\nCMD ["-e", "setInterval(()=>{},1000)"]\n')
docker('build', '-t', repository + ':fixture', work)
docker('push', repository + ':fixture')
digest = json.loads(docker('image', 'inspect', repository + ':fixture'))[0]['RepoDigests'][0].split('@')[1]
images = {name: {'name': name, 'repository': repository + '-' + name, 'digest': digest} for name in ('backend', 'frontend')}
for image in images.values():
docker('tag', repository + ':fixture', image['repository'] + ':fixture')
docker('push', image['repository'] + ':fixture')
# Copy health, networking and SDK settings from the original fixture, but start the new image's migration script.
updater.releases = Releases(config, images)
for version in ('3.4.0', '3.5.0'):
http.version = version
checked = updater.execute('check', version)
assert checked['plan']['storageProvider'] == 'seaweedfs'
assert not checked['plan']['requiresComposeAdoption']
result = updater.execute('apply', version, expected_fingerprint=checked['plan']['approvalFingerprint'])
assert result['status'] == 'succeeded'
assert not migration.inspect(migration.state['containers']['minio']['id'])['State']['Running']
old_matrix = migration.inspect(retained)
assert not old_matrix['State']['Running'] and old_matrix['HostConfig']['RestartPolicy']['Name'] == 'no'
backup = Path(result['backup'])
with tarfile.open(backup / 'files.tar') as archive:
assert archive.extractfile('matrix/postgres/retained.txt').read() == b'retired Matrix'
assert 'storage/config/s3.json' in archive.getnames()
assert 'minio' not in archive.getnames()
metadata = u.read_json(backup / 'complete.json')
for name, expected in metadata['sha256'].items():
assert u.file_sha(backup / name) == expected
print(json.dumps({'phase': 'update_complete', 'version': version, 'backup': str(backup)}), flush=True)
# Restore the *updater's* cold archive, not the original migration backup.
restore = root / '.updates/update-restore-test'
restore.mkdir()
subprocess.run(['tar', '-xpf', str(backup / 'files.tar'), '-C', str(restore)], check=True)
store = updater.storage['service']
restore_name = project + '-update-restore'
try:
docker('run', '-d', '--pull', 'never', '--name', restore_name, '--network', project + '-internal',
'--label', 'com.docker.compose.project=' + project,
'-v', str(restore / 'storage/seaweedfs') + ':/data', '-v', str(restore / 'storage/config') + ':/etc/fedeo:ro',
'-e', 'FEDEO_S3_CONFIG_SHA256=' + updater.storage['configSha256']['s3.json'],
'--entrypoint', store['entrypoint'][0], store['image'], *store['entrypoint'][1:], *store['command'])
migration.helper('restore-verify', endpoint='http://' + restore_name + ':9000')
finally:
docker('rm', '-f', restore_name)
print(json.dumps({'result': 'passed', 'restoredUpdaterBackup': str(backup), 'evidence': str(root)}), flush=True)
except BaseException:
print('Failure evidence: ' + str(root), flush=True)
for path in sorted((root / '.updates').glob('*/commands.log')):
print(str(path) + '\n' + path.read_text()[-5000:], flush=True)
raise
finally:
ids = docker('ps', '-aq', '--filter', 'label=com.docker.compose.project=' + project).split()
if ids:
docker('rm', '-f', '-v', *ids)
if registry_id:
docker('rm', '-f', '-v', registry_id)
docker('network', 'rm', project + '-internal')
if __name__ == '__main__':
run()

View File

@@ -70,6 +70,16 @@ class UnixConnection(http.client.HTTPConnection):
class ServiceTests(unittest.TestCase):
def test_storage_fields_are_sanitized_and_new_phases_remain_compatible(self):
plan = s.public_plan({'storageProvider': 'seaweedfs', 'storageImageRetained': True,
'storageBackupPaths': ['/private/storage'], 'service': {'secret': 'hidden'}})
self.assertEqual(plan['storageProvider'], 'seaweedfs')
self.assertTrue(plan['storageImageRetained'])
self.assertNotIn('private', json.dumps(plan))
self.assertNotIn('hidden', json.dumps(plan))
self.assertEqual(s.public_state({'phase': 'checking_storage'})['phase'], 'validating')
self.assertEqual(s.public_state({'phase': 'storage_adopted'})['phase'], 'resolved')
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name)

View File

@@ -0,0 +1,278 @@
"""Provider handover and update failure tests using real Compose parsing."""
import copy
import json
from pathlib import Path
import subprocess
import tarfile
import unittest
from test_updater import ROOT, Runner, u
import test_updater as base
class StorageRunner(Runner):
def __init__(self, log, http):
super().__init__(log, http)
self.storage = None
self.old_minio = None
self.db_image = 'sha256:' + 'd' * 64
def run(self, args, **kwargs):
if args[:2] == ['docker', 'inspect'] and args[2] in ('objectstore', 'minio', 'retained-minio', 'db'):
self.calls.append(args)
if args[2] == 'db':
return json.dumps([{'Image': self.db_image, 'Config': {'Labels': {'com.docker.compose.service': 'db'}},
'State': {'Running': True, 'Health': {'Status': 'healthy'}}}])
return json.dumps([self.storage if args[2] == 'objectstore' else self.old_minio])
if args[:3] == ['docker', 'image', 'inspect'] and u.IMAGE_ID.fullmatch(args[3]):
self.calls.append(args)
return args[3] if '--format' in args else json.dumps([{'Id': args[3], 'Size': 1000}])
if args[:3] == ['docker', 'image', 'save']:
self.calls.append(args)
Path(args[args.index('-o') + 1]).write_bytes(b'preserved storage image')
return ''
if args[:2] == ['docker', 'compose']:
if 'ps' in args and args[-1] in ('objectstore', 'traefik', 'node-exporter'):
self.calls.append(args)
return args[-1]
if 'stop' in args and 'objectstore' in args:
self.storage['State']['Running'] = False
if 'start' in args or 'up' in args:
if 'objectstore' in args or ('up' in args and '--no-deps' not in args):
self.storage['State']['Running'] = True
result = super().run(args, **kwargs)
if 'ps' in args and 'backend\nfrontend' in result:
result += '\nobjectstore'
return result
class StorageTests(unittest.TestCase):
setUp = base.UpdateTests.setUp
tearDown = base.UpdateTests.tearDown
called = base.UpdateTests.called
run_update = base.UpdateTests.run_update
def migrated(self):
self.runner = StorageRunner(self.root / 'log', self.http)
self.updater.runner_factory = lambda log: self.runner
self.job = self.root / '.updates/storage-migration'
self.job.mkdir()
(self.root / 'storage/config').mkdir(parents=True)
(self.root / 'storage/seaweedfs').mkdir()
for name in ('s3.json', 'start-seaweedfs.sh'):
(self.root / 'storage/config' / name).write_text('protected-' + name)
(self.root / 'storage/seaweedfs/data').write_bytes(b'complete volume data')
original = u.compose_strings(json.loads(subprocess.check_output([
'docker', 'compose', '--project-directory', str(self.root), '--env-file', str(self.root / '.env'),
'-f', str(self.root / 'docker-compose.yml'), 'config', '--format', 'json'])))
u.write_json(self.job / 'original.json', original)
target = copy.deepcopy(original)
services = target['services']
services.pop('minio'); services.pop('createbuckets')
services['db']['image'] = self.runner.db_image
services['backend']['environment'].update(AWS_REQUEST_CHECKSUM_CALCULATION='WHEN_REQUIRED', FEDEO_RUN_MIGRATIONS='false')
services['backend']['depends_on'] = {'db': {'condition': 'service_healthy'}, 'objectstore': {'condition': 'service_healthy'}}
weed = u.read_json(ROOT / 'scripts/storage/seaweedfs.json')
weed.update(image='sha256:' + 'e' * 64, container_name='fedeo-objectstore', restart='unless-stopped',
networks={'internal': {'aliases': ['minio']}}, user='0:0',
environment={'FEDEO_S3_CONFIG_SHA256': u.file_sha(self.root / 'storage/config/s3.json')},
volumes=[{'type': 'bind', 'source': str(self.root / 'storage/seaweedfs'), 'target': '/data'},
{'type': 'bind', 'source': str(self.root / 'storage/config'), 'target': '/etc/fedeo', 'read_only': True}])
services['objectstore'] = weed
u.write_json(self.job / 'target.json', u.compose_strings(target, escape=True))
(self.root / 'docker-compose.yml').write_bytes((self.job / 'target.json').read_bytes())
network = original['networks']['internal']['name']
self.runner.storage = {'Id': 'objectstore', 'Image': weed['image'],
'Config': {'Labels': {'com.docker.compose.service': 'objectstore'},
'Env': ['FEDEO_S3_CONFIG_SHA256=' + weed['environment']['FEDEO_S3_CONFIG_SHA256']]},
'State': {'Running': True, 'Health': {'Status': 'healthy'}}, 'HostConfig': {},
'NetworkSettings': {'Networks': {network: {'Aliases': ['minio']}}},
'Mounts': [{'Type': 'bind', 'Source': v['source'], 'Destination': v['target'], 'RW': not v.get('read_only')} for v in weed['volumes']]}
self.runner.old_minio = {'Id': 'retained-minio', 'Image': 'image-minio',
'Config': {'Labels': {'com.docker.compose.service': 'minio'}}, 'State': {'Running': False},
'HostConfig': {'RestartPolicy': {'Name': 'no'}}, 'NetworkSettings': {'Networks': {}}}
self.runner.local_images.update([weed['image'], self.runner.db_image])
state = {'schemaVersion': 1, 'phase': 'complete', 'provider': 'seaweedfs', 'writesReleased': True,
'version': '3.0.0', 'restoreVerifiedAt': u.now(), 'weedImage': weed['image'], 'network': network,
'fingerprint': {'.env': u.file_sha(self.root / '.env')}, 'count': 1, 'bytes': 20,
'containers': {name: {'id': name, 'image': 'image-' + name} for name in u.CORE_SERVICES | {'minio'}}}
state['containers']['db']['image'] = self.runner.db_image
state['containers']['minio']['id'] = 'retained-minio'
u.write_json(self.job / 'copy/verified.json', {'count': 1, 'bytes': 20, 'verifiedAt': u.now(), 'objects': [{}]})
for name in ('restore-contract', 'acceptance', 'backend-acceptance'):
u.write_json(self.job / name / 'result.json', {'operations': ['PUT', 'HEAD', 'GET', 'DELETE', 'SigV4', 'presigned-24h', 'anonymous-denied', 'wrong-key-denied']})
u.write_json(self.job / 'backup/complete.json', {})
(self.job / 'backup/seaweedfs.tar').write_bytes(b'verified cold restore archive')
state['seaweedBackupSha256'] = u.file_sha(self.job / 'backup/seaweedfs.tar')
u.write_json(self.job / 'state.json', state)
u.write_json(self.root / '.updates/status.json', {'status': 'failed', 'phase': 'storage_migration',
'job': str(self.job), 'recoveryRequired': True})
return state
def adopt(self):
self.migrated()
self.assertEqual(self.updater.adopt_storage()['status'], 'adopted')
self.assertFalse(self.updater.status()['recoveryRequired'])
self.runner.calls.clear()
def test_adoption_is_read_only_for_services_and_idempotent(self):
self.adopt()
self.assertEqual(self.updater.adopt_storage()['status'], 'already_adopted')
self.assertFalse(self.called('stop'))
self.assertFalse(self.called('pull'))
result = self.updater.execute('check', '3.1.0')
self.assertEqual(result['plan']['storageProvider'], 'seaweedfs')
self.assertFalse(result['plan']['requiresComposeAdoption'])
self.assertEqual(result['plan']['storageBackupPaths'], ['storage'])
def test_incomplete_migration_cannot_be_adopted_or_resolved(self):
state = self.migrated()
for fields in ({'phase': 'released'}, {'writesReleased': False}, {'restoreVerifiedAt': None}):
u.write_json(self.job / 'state.json', {**state, **fields})
with self.assertRaisesRegex(u.UpdateError, 'vollständig'):
self.updater.adopt_storage()
self.assertTrue(self.updater.status()['recoveryRequired'])
with self.assertRaisesRegex(u.UpdateError, 'adopt-storage'):
self.updater.resolve('3.0.0')
with self.assertRaisesRegex(u.UpdateError, 'adopt-storage'):
self.run_update()
def test_adoption_rejects_modified_backup_missing_contract_and_changed_environment(self):
self.migrated()
for path in (self.job / 'backup/seaweedfs.tar', self.job / 'backend-acceptance/result.json', self.root / '.env'):
content = path.read_bytes()
path.write_text('{}')
with self.assertRaises(u.UpdateError):
self.updater.adopt_storage()
self.assertTrue(self.updater.status()['recoveryRequired'])
path.write_bytes(content)
self.assertFalse((self.root / '.updates/storage-provider.json').exists())
def test_update_retains_storage_and_database_and_backs_up_all_storage(self):
self.adopt()
self.runner.real_tar = True
result = self.updater.execute('apply', '3.1.0')
self.assertEqual(result['status'], 'succeeded')
self.assertFalse(self.runner.old_minio['State']['Running'])
self.assertTrue(self.runner.storage['State']['Running'])
self.assertFalse(any('minio' in image or 'seaweed' in image or u.IMAGE_ID.fullmatch(image) for image in self.runner.downloads))
self.assertFalse(any('createbuckets' in a for a in self.runner.calls if 'run' in a))
backup = Path(result['backup'])
with tarfile.open(backup / 'files.tar') as archive:
self.assertEqual(archive.extractfile('storage/seaweedfs/data').read(), b'complete volume data')
self.assertIn('storage/config/s3.json', archive.getnames())
self.assertNotIn('minio', archive.getnames())
self.assertTrue((backup / 'storage-image.tar').is_file())
self.assertTrue((backup / 'config/.updates/storage-provider.json').is_file())
calls = self.runner.calls
contract = next(i for i, a in enumerate(calls) if u.S3_UPGRADE_CHECK in a)
self.assertLess(contract, next(i for i, a in enumerate(calls) if 'migrate' in a))
# A later update also preserves the provider, with no new adoption required.
(self.root / '.env').write_bytes(u.set_version((self.root / '.env').read_bytes(), '3.0.0'))
target = u.read_json(self.root / 'docker-compose.yml')
target['services']['backend']['environment']['FEDEO_VERSION'] = '3.0.0'
u.write_json(self.root / 'docker-compose.yml', target)
u.write_json(self.root / '.updates/installed.json', {'composeSha256': u.file_sha(self.root / 'docker-compose.yml')})
self.assertEqual(self.updater.execute('apply', '3.1.0')['status'], 'succeeded')
def test_storage_drift_and_restarted_minio_block_before_maintenance(self):
self.adopt()
original = self.runner.old_minio['State'].copy()
self.runner.old_minio['State']['Running'] = True
with self.assertRaisesRegex(u.UpdateError, 'stillgelegt'):
self.run_update()
self.runner.old_minio['State'] = original
(self.root / 'storage/config/s3.json').write_text('changed')
with self.assertRaisesRegex(u.UpdateError, 'Konfiguration'):
self.run_update()
self.assertFalse(self.called('stop'))
def test_storage_config_changed_during_pull_blocks_before_maintenance(self):
self.adopt()
self.runner.on_pull = lambda: (self.root / 'storage/config/s3.json').write_text('changed')
with self.assertRaisesRegex(u.UpdateError, 'während'):
self.run_update()
self.assertFalse(self.called('stop'))
def test_backup_failure_restarts_seaweed_without_restarting_minio(self):
self.adopt()
self.runner.fail = lambda a: a[0] == 'tar' and '-cpf' in a
with self.assertRaises(u.UpdateError):
self.run_update()
self.assertFalse(self.updater.status()['recoveryRequired'])
start = next(a for a in self.runner.calls if 'start' in a)
self.assertIn('objectstore', start)
self.assertNotIn('minio', start)
def test_storage_must_remain_stopped_during_cold_backup(self):
self.adopt()
self.runner.on_stop = lambda: self.runner.storage['State'].update(Running=True)
with self.assertRaisesRegex(u.UpdateError, 'kalten Sicherung'):
self.run_update()
self.assertFalse(self.called('migrate'))
def test_target_s3_failure_blocks_database_migrations_and_retains_recovery(self):
self.adopt()
self.runner.fail = lambda a: u.S3_UPGRADE_CHECK in a
with self.assertRaises(u.UpdateError):
self.run_update()
self.assertTrue(self.updater.status()['recoveryRequired'])
self.assertFalse(self.called('migrate'))
self.assertTrue(self.updater.status()['storageProbeKey'].startswith('.fedeo-update-check/'))
self.runner.fail = None
self.assertEqual(self.updater.resolve('3.1.0')['status'], 'recovered')
self.assertTrue(self.called(u.S3_CLEANUP_CHECK))
self.assertIsNone(self.updater.status()['storageProbeKey'])
def test_symlink_and_changed_database_major_are_rejected(self):
self.adopt()
link = self.root / 'storage/seaweedfs/external'
link.symlink_to(self.root / 'minio', target_is_directory=True)
with self.assertRaisesRegex(u.UpdateError, 'Links'):
self.run_update()
link.unlink()
self.http.files['docker-compose.selfhost.yml'] = self.http.files['docker-compose.selfhost.yml'].replace(b'postgres:16', b'postgres:17')
self.http.refresh()
with self.assertRaisesRegex(u.UpdateError, 'PostgreSQL'):
self.run_update()
self.assertFalse(self.called('stop'))
def test_literal_dollars_survive_generated_compose(self):
self.adopt()
value = 'secret-$FOO-$$-with-dollar'
config = u.compose_strings(u.read_json(self.root / 'docker-compose.yml'))
config['services']['backend']['environment']['COOKIE_SECRET'] = value
u.write_json(self.root / 'docker-compose.yml', u.compose_strings(config, escape=True))
text = (self.root / '.env').read_text()
import re
(self.root / '.env').write_text(re.sub(r'(?m)^COOKIE_SECRET=.*$', "COOKIE_SECRET='" + value + "'", text))
result = self.run_update()
self.assertEqual(result['status'], 'succeeded')
self.assertEqual(self.updater.candidate['services']['backend']['environment']['COOKIE_SECRET'], value)
def test_standard_installer_cannot_overwrite_a_planned_or_migrated_instance(self):
import os
original = (self.root / 'docker-compose.yml').read_bytes()
(self.root / 'scripts').mkdir()
setup = self.root / 'scripts/selfhost-setup.sh'
setup.write_bytes((ROOT / 'scripts/selfhost-setup.sh').read_bytes())
for relative in ('storage', '.updates/storage-provider.json', '.updates/storage-migration/state.json'):
path = self.root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('migration marker')
result = subprocess.run(['bash', str(ROOT / 'scripts/selfhost-install.sh')], capture_output=True, text=True,
env={**os.environ, 'FEDEO_DIR': str(self.root), 'FEDEO_TTY_INPUT': '/dev/null'})
self.assertEqual(result.returncode, 1)
self.assertIn('nicht überschreiben', result.stderr)
self.assertEqual((self.root / 'docker-compose.yml').read_bytes(), original)
for args in (['--force', '--no-start'], ['--uninstall', '--purge']):
result = subprocess.run(['bash', str(setup), *args], capture_output=True, text=True,
env={**os.environ, 'FEDEO_TTY_INPUT': '/dev/null'})
self.assertEqual(result.returncode, 1)
self.assertIn('nicht überschreiben oder entfernen', result.stderr)
self.assertTrue((self.root / '.env').is_file())
path.unlink()
if __name__ == '__main__':
unittest.main()

View File

@@ -138,6 +138,8 @@ class Runner:
return result.stdout
if 'ps' in args:
if args[:2] == ['docker', 'ps']:
if any(a.startswith('label=com.docker.compose.project=') for a in args):
return 'backend\nfrontend\ndb\nminio\ntraefik\nnode-exporter\ncreatebuckets\n' + '\n'.join(self.matrix_containers)
return ''
if args[-1] in ('backend', 'frontend', 'db', 'minio') or args[-1] in self.matrix_containers:
return args[-1]