279 lines
16 KiB
Python
279 lines
16 KiB
Python
"""Provider handover and update failure tests using real Compose parsing."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tarfile
|
|
import unittest
|
|
|
|
from test_updater import ROOT, Runner, u
|
|
import test_updater as base
|
|
|
|
|
|
class StorageRunner(Runner):
|
|
def __init__(self, log, http):
|
|
super().__init__(log, http)
|
|
self.storage = None
|
|
self.old_minio = None
|
|
self.db_image = 'sha256:' + 'd' * 64
|
|
|
|
def run(self, args, **kwargs):
|
|
if args[:2] == ['docker', 'inspect'] and args[2] in ('objectstore', 'minio', 'retained-minio', 'db'):
|
|
self.calls.append(args)
|
|
if args[2] == 'db':
|
|
return json.dumps([{'Image': self.db_image, 'Config': {'Labels': {'com.docker.compose.service': 'db'}},
|
|
'State': {'Running': True, 'Health': {'Status': 'healthy'}}}])
|
|
return json.dumps([self.storage if args[2] == 'objectstore' else self.old_minio])
|
|
if args[:3] == ['docker', 'image', 'inspect'] and u.IMAGE_ID.fullmatch(args[3]):
|
|
self.calls.append(args)
|
|
return args[3] if '--format' in args else json.dumps([{'Id': args[3], 'Size': 1000}])
|
|
if args[:3] == ['docker', 'image', 'save']:
|
|
self.calls.append(args)
|
|
Path(args[args.index('-o') + 1]).write_bytes(b'preserved storage image')
|
|
return ''
|
|
if args[:2] == ['docker', 'compose']:
|
|
if 'ps' in args and args[-1] in ('objectstore', 'traefik', 'node-exporter'):
|
|
self.calls.append(args)
|
|
return args[-1]
|
|
if 'stop' in args and 'objectstore' in args:
|
|
self.storage['State']['Running'] = False
|
|
if 'start' in args or 'up' in args:
|
|
if 'objectstore' in args or ('up' in args and '--no-deps' not in args):
|
|
self.storage['State']['Running'] = True
|
|
result = super().run(args, **kwargs)
|
|
if 'ps' in args and 'backend\nfrontend' in result:
|
|
result += '\nobjectstore'
|
|
return result
|
|
|
|
|
|
class StorageTests(unittest.TestCase):
|
|
setUp = base.UpdateTests.setUp
|
|
tearDown = base.UpdateTests.tearDown
|
|
called = base.UpdateTests.called
|
|
run_update = base.UpdateTests.run_update
|
|
|
|
def migrated(self):
|
|
self.runner = StorageRunner(self.root / 'log', self.http)
|
|
self.updater.runner_factory = lambda log: self.runner
|
|
self.job = self.root / '.updates/storage-migration'
|
|
self.job.mkdir()
|
|
(self.root / 'storage/config').mkdir(parents=True)
|
|
(self.root / 'storage/seaweedfs').mkdir()
|
|
for name in ('s3.json', 'start-seaweedfs.sh'):
|
|
(self.root / 'storage/config' / name).write_text('protected-' + name)
|
|
(self.root / 'storage/seaweedfs/data').write_bytes(b'complete volume data')
|
|
original = u.compose_strings(json.loads(subprocess.check_output([
|
|
'docker', 'compose', '--project-directory', str(self.root), '--env-file', str(self.root / '.env'),
|
|
'-f', str(self.root / 'docker-compose.yml'), 'config', '--format', 'json'])))
|
|
u.write_json(self.job / 'original.json', original)
|
|
target = copy.deepcopy(original)
|
|
services = target['services']
|
|
services.pop('minio'); services.pop('createbuckets')
|
|
services['db']['image'] = self.runner.db_image
|
|
services['backend']['environment'].update(AWS_REQUEST_CHECKSUM_CALCULATION='WHEN_REQUIRED', FEDEO_RUN_MIGRATIONS='false')
|
|
services['backend']['depends_on'] = {'db': {'condition': 'service_healthy'}, 'objectstore': {'condition': 'service_healthy'}}
|
|
weed = u.read_json(ROOT / 'scripts/storage/seaweedfs.json')
|
|
weed.update(image='sha256:' + 'e' * 64, container_name='fedeo-objectstore', restart='unless-stopped',
|
|
networks={'internal': {'aliases': ['minio']}}, user='0:0',
|
|
environment={'FEDEO_S3_CONFIG_SHA256': u.file_sha(self.root / 'storage/config/s3.json')},
|
|
volumes=[{'type': 'bind', 'source': str(self.root / 'storage/seaweedfs'), 'target': '/data'},
|
|
{'type': 'bind', 'source': str(self.root / 'storage/config'), 'target': '/etc/fedeo', 'read_only': True}])
|
|
services['objectstore'] = weed
|
|
u.write_json(self.job / 'target.json', u.compose_strings(target, escape=True))
|
|
(self.root / 'docker-compose.yml').write_bytes((self.job / 'target.json').read_bytes())
|
|
network = original['networks']['internal']['name']
|
|
self.runner.storage = {'Id': 'objectstore', 'Image': weed['image'],
|
|
'Config': {'Labels': {'com.docker.compose.service': 'objectstore'},
|
|
'Env': ['FEDEO_S3_CONFIG_SHA256=' + weed['environment']['FEDEO_S3_CONFIG_SHA256']]},
|
|
'State': {'Running': True, 'Health': {'Status': 'healthy'}}, 'HostConfig': {},
|
|
'NetworkSettings': {'Networks': {network: {'Aliases': ['minio']}}},
|
|
'Mounts': [{'Type': 'bind', 'Source': v['source'], 'Destination': v['target'], 'RW': not v.get('read_only')} for v in weed['volumes']]}
|
|
self.runner.old_minio = {'Id': 'retained-minio', 'Image': 'image-minio',
|
|
'Config': {'Labels': {'com.docker.compose.service': 'minio'}}, 'State': {'Running': False},
|
|
'HostConfig': {'RestartPolicy': {'Name': 'no'}}, 'NetworkSettings': {'Networks': {}}}
|
|
self.runner.local_images.update([weed['image'], self.runner.db_image])
|
|
state = {'schemaVersion': 1, 'phase': 'complete', 'provider': 'seaweedfs', 'writesReleased': True,
|
|
'version': '3.0.0', 'restoreVerifiedAt': u.now(), 'weedImage': weed['image'], 'network': network,
|
|
'fingerprint': {'.env': u.file_sha(self.root / '.env')}, 'count': 1, 'bytes': 20,
|
|
'containers': {name: {'id': name, 'image': 'image-' + name} for name in u.CORE_SERVICES | {'minio'}}}
|
|
state['containers']['db']['image'] = self.runner.db_image
|
|
state['containers']['minio']['id'] = 'retained-minio'
|
|
u.write_json(self.job / 'copy/verified.json', {'count': 1, 'bytes': 20, 'verifiedAt': u.now(), 'objects': [{}]})
|
|
for name in ('restore-contract', 'acceptance', 'backend-acceptance'):
|
|
u.write_json(self.job / name / 'result.json', {'operations': ['PUT', 'HEAD', 'GET', 'DELETE', 'SigV4', 'presigned-24h', 'anonymous-denied', 'wrong-key-denied']})
|
|
u.write_json(self.job / 'backup/complete.json', {})
|
|
(self.job / 'backup/seaweedfs.tar').write_bytes(b'verified cold restore archive')
|
|
state['seaweedBackupSha256'] = u.file_sha(self.job / 'backup/seaweedfs.tar')
|
|
u.write_json(self.job / 'state.json', state)
|
|
u.write_json(self.root / '.updates/status.json', {'status': 'failed', 'phase': 'storage_migration',
|
|
'job': str(self.job), 'recoveryRequired': True})
|
|
return state
|
|
|
|
def adopt(self):
|
|
self.migrated()
|
|
self.assertEqual(self.updater.adopt_storage()['status'], 'adopted')
|
|
self.assertFalse(self.updater.status()['recoveryRequired'])
|
|
self.runner.calls.clear()
|
|
|
|
def test_adoption_is_read_only_for_services_and_idempotent(self):
|
|
self.adopt()
|
|
self.assertEqual(self.updater.adopt_storage()['status'], 'already_adopted')
|
|
self.assertFalse(self.called('stop'))
|
|
self.assertFalse(self.called('pull'))
|
|
result = self.updater.execute('check', '3.1.0')
|
|
self.assertEqual(result['plan']['storageProvider'], 'seaweedfs')
|
|
self.assertFalse(result['plan']['requiresComposeAdoption'])
|
|
self.assertEqual(result['plan']['storageBackupPaths'], ['storage'])
|
|
|
|
def test_incomplete_migration_cannot_be_adopted_or_resolved(self):
|
|
state = self.migrated()
|
|
for fields in ({'phase': 'released'}, {'writesReleased': False}, {'restoreVerifiedAt': None}):
|
|
u.write_json(self.job / 'state.json', {**state, **fields})
|
|
with self.assertRaisesRegex(u.UpdateError, 'vollständig'):
|
|
self.updater.adopt_storage()
|
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
|
with self.assertRaisesRegex(u.UpdateError, 'adopt-storage'):
|
|
self.updater.resolve('3.0.0')
|
|
with self.assertRaisesRegex(u.UpdateError, 'adopt-storage'):
|
|
self.run_update()
|
|
|
|
def test_adoption_rejects_modified_backup_missing_contract_and_changed_environment(self):
|
|
self.migrated()
|
|
for path in (self.job / 'backup/seaweedfs.tar', self.job / 'backend-acceptance/result.json', self.root / '.env'):
|
|
content = path.read_bytes()
|
|
path.write_text('{}')
|
|
with self.assertRaises(u.UpdateError):
|
|
self.updater.adopt_storage()
|
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
|
path.write_bytes(content)
|
|
self.assertFalse((self.root / '.updates/storage-provider.json').exists())
|
|
|
|
def test_update_retains_storage_and_database_and_backs_up_all_storage(self):
|
|
self.adopt()
|
|
self.runner.real_tar = True
|
|
result = self.updater.execute('apply', '3.1.0')
|
|
self.assertEqual(result['status'], 'succeeded')
|
|
self.assertFalse(self.runner.old_minio['State']['Running'])
|
|
self.assertTrue(self.runner.storage['State']['Running'])
|
|
self.assertFalse(any('minio' in image or 'seaweed' in image or u.IMAGE_ID.fullmatch(image) for image in self.runner.downloads))
|
|
self.assertFalse(any('createbuckets' in a for a in self.runner.calls if 'run' in a))
|
|
backup = Path(result['backup'])
|
|
with tarfile.open(backup / 'files.tar') as archive:
|
|
self.assertEqual(archive.extractfile('storage/seaweedfs/data').read(), b'complete volume data')
|
|
self.assertIn('storage/config/s3.json', archive.getnames())
|
|
self.assertNotIn('minio', archive.getnames())
|
|
self.assertTrue((backup / 'storage-image.tar').is_file())
|
|
self.assertTrue((backup / 'config/.updates/storage-provider.json').is_file())
|
|
calls = self.runner.calls
|
|
contract = next(i for i, a in enumerate(calls) if u.S3_UPGRADE_CHECK in a)
|
|
self.assertLess(contract, next(i for i, a in enumerate(calls) if 'migrate' in a))
|
|
# A later update also preserves the provider, with no new adoption required.
|
|
(self.root / '.env').write_bytes(u.set_version((self.root / '.env').read_bytes(), '3.0.0'))
|
|
target = u.read_json(self.root / 'docker-compose.yml')
|
|
target['services']['backend']['environment']['FEDEO_VERSION'] = '3.0.0'
|
|
u.write_json(self.root / 'docker-compose.yml', target)
|
|
u.write_json(self.root / '.updates/installed.json', {'composeSha256': u.file_sha(self.root / 'docker-compose.yml')})
|
|
self.assertEqual(self.updater.execute('apply', '3.1.0')['status'], 'succeeded')
|
|
|
|
def test_storage_drift_and_restarted_minio_block_before_maintenance(self):
|
|
self.adopt()
|
|
original = self.runner.old_minio['State'].copy()
|
|
self.runner.old_minio['State']['Running'] = True
|
|
with self.assertRaisesRegex(u.UpdateError, 'stillgelegt'):
|
|
self.run_update()
|
|
self.runner.old_minio['State'] = original
|
|
(self.root / 'storage/config/s3.json').write_text('changed')
|
|
with self.assertRaisesRegex(u.UpdateError, 'Konfiguration'):
|
|
self.run_update()
|
|
self.assertFalse(self.called('stop'))
|
|
|
|
def test_storage_config_changed_during_pull_blocks_before_maintenance(self):
|
|
self.adopt()
|
|
self.runner.on_pull = lambda: (self.root / 'storage/config/s3.json').write_text('changed')
|
|
with self.assertRaisesRegex(u.UpdateError, 'während'):
|
|
self.run_update()
|
|
self.assertFalse(self.called('stop'))
|
|
|
|
def test_backup_failure_restarts_seaweed_without_restarting_minio(self):
|
|
self.adopt()
|
|
self.runner.fail = lambda a: a[0] == 'tar' and '-cpf' in a
|
|
with self.assertRaises(u.UpdateError):
|
|
self.run_update()
|
|
self.assertFalse(self.updater.status()['recoveryRequired'])
|
|
start = next(a for a in self.runner.calls if 'start' in a)
|
|
self.assertIn('objectstore', start)
|
|
self.assertNotIn('minio', start)
|
|
|
|
def test_storage_must_remain_stopped_during_cold_backup(self):
|
|
self.adopt()
|
|
self.runner.on_stop = lambda: self.runner.storage['State'].update(Running=True)
|
|
with self.assertRaisesRegex(u.UpdateError, 'kalten Sicherung'):
|
|
self.run_update()
|
|
self.assertFalse(self.called('migrate'))
|
|
|
|
def test_target_s3_failure_blocks_database_migrations_and_retains_recovery(self):
|
|
self.adopt()
|
|
self.runner.fail = lambda a: u.S3_UPGRADE_CHECK in a
|
|
with self.assertRaises(u.UpdateError):
|
|
self.run_update()
|
|
self.assertTrue(self.updater.status()['recoveryRequired'])
|
|
self.assertFalse(self.called('migrate'))
|
|
self.assertTrue(self.updater.status()['storageProbeKey'].startswith('.fedeo-update-check/'))
|
|
self.runner.fail = None
|
|
self.assertEqual(self.updater.resolve('3.1.0')['status'], 'recovered')
|
|
self.assertTrue(self.called(u.S3_CLEANUP_CHECK))
|
|
self.assertIsNone(self.updater.status()['storageProbeKey'])
|
|
|
|
def test_symlink_and_changed_database_major_are_rejected(self):
|
|
self.adopt()
|
|
link = self.root / 'storage/seaweedfs/external'
|
|
link.symlink_to(self.root / 'minio', target_is_directory=True)
|
|
with self.assertRaisesRegex(u.UpdateError, 'Links'):
|
|
self.run_update()
|
|
link.unlink()
|
|
self.http.files['docker-compose.selfhost.yml'] = self.http.files['docker-compose.selfhost.yml'].replace(b'postgres:16', b'postgres:17')
|
|
self.http.refresh()
|
|
with self.assertRaisesRegex(u.UpdateError, 'PostgreSQL'):
|
|
self.run_update()
|
|
self.assertFalse(self.called('stop'))
|
|
|
|
def test_literal_dollars_survive_generated_compose(self):
|
|
self.adopt()
|
|
value = 'secret-$FOO-$$-with-dollar'
|
|
config = u.compose_strings(u.read_json(self.root / 'docker-compose.yml'))
|
|
config['services']['backend']['environment']['COOKIE_SECRET'] = value
|
|
u.write_json(self.root / 'docker-compose.yml', u.compose_strings(config, escape=True))
|
|
text = (self.root / '.env').read_text()
|
|
import re
|
|
(self.root / '.env').write_text(re.sub(r'(?m)^COOKIE_SECRET=.*$', "COOKIE_SECRET='" + value + "'", text))
|
|
result = self.run_update()
|
|
self.assertEqual(result['status'], 'succeeded')
|
|
self.assertEqual(self.updater.candidate['services']['backend']['environment']['COOKIE_SECRET'], value)
|
|
|
|
def test_standard_installer_cannot_overwrite_a_planned_or_migrated_instance(self):
|
|
import os
|
|
original = (self.root / 'docker-compose.yml').read_bytes()
|
|
(self.root / 'scripts').mkdir()
|
|
setup = self.root / 'scripts/selfhost-setup.sh'
|
|
setup.write_bytes((ROOT / 'scripts/selfhost-setup.sh').read_bytes())
|
|
for relative in ('storage', '.updates/storage-provider.json', '.updates/storage-migration/state.json'):
|
|
path = self.root / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text('migration marker')
|
|
result = subprocess.run(['bash', str(ROOT / 'scripts/selfhost-install.sh')], capture_output=True, text=True,
|
|
env={**os.environ, 'FEDEO_DIR': str(self.root), 'FEDEO_TTY_INPUT': '/dev/null'})
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('nicht überschreiben', result.stderr)
|
|
self.assertEqual((self.root / 'docker-compose.yml').read_bytes(), original)
|
|
for args in (['--force', '--no-start'], ['--uninstall', '--purge']):
|
|
result = subprocess.run(['bash', str(setup), *args], capture_output=True, text=True,
|
|
env={**os.environ, 'FEDEO_TTY_INPUT': '/dev/null'})
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('nicht überschreiben oder entfernen', result.stderr)
|
|
self.assertTrue((self.root / '.env').is_file())
|
|
path.unlink()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|