"""Provider handover and update failure tests using real Compose parsing.""" import copy import json from pathlib import Path import subprocess import tarfile import unittest from test_updater import ROOT, Runner, u import test_updater as base class StorageRunner(Runner): def __init__(self, log, http): super().__init__(log, http) self.storage = None self.old_minio = None self.db_image = 'sha256:' + 'd' * 64 def run(self, args, **kwargs): if args[:2] == ['docker', 'inspect'] and args[2] in ('objectstore', 'minio', 'retained-minio', 'db'): self.calls.append(args) if args[2] == 'db': return json.dumps([{'Image': self.db_image, 'Config': {'Labels': {'com.docker.compose.service': 'db'}}, 'State': {'Running': True, 'Health': {'Status': 'healthy'}}}]) return json.dumps([self.storage if args[2] == 'objectstore' else self.old_minio]) if args[:3] == ['docker', 'image', 'inspect'] and u.IMAGE_ID.fullmatch(args[3]): self.calls.append(args) return args[3] if '--format' in args else json.dumps([{'Id': args[3], 'Size': 1000}]) if args[:3] == ['docker', 'image', 'save']: self.calls.append(args) Path(args[args.index('-o') + 1]).write_bytes(b'preserved storage image') return '' if args[:2] == ['docker', 'compose']: if 'ps' in args and args[-1] in ('objectstore', 'traefik', 'node-exporter'): self.calls.append(args) return args[-1] if 'stop' in args and 'objectstore' in args: self.storage['State']['Running'] = False if 'start' in args or 'up' in args: if 'objectstore' in args or ('up' in args and '--no-deps' not in args): self.storage['State']['Running'] = True result = super().run(args, **kwargs) if 'ps' in args and 'backend\nfrontend' in result: result += '\nobjectstore' return result class StorageTests(unittest.TestCase): setUp = base.UpdateTests.setUp tearDown = base.UpdateTests.tearDown called = base.UpdateTests.called run_update = base.UpdateTests.run_update def migrated(self): self.runner = StorageRunner(self.root / 'log', self.http) self.updater.runner_factory = lambda log: self.runner self.job = self.root / '.updates/storage-migration' self.job.mkdir() (self.root / 'storage/config').mkdir(parents=True) (self.root / 'storage/seaweedfs').mkdir() for name in ('s3.json', 'start-seaweedfs.sh'): (self.root / 'storage/config' / name).write_text('protected-' + name) (self.root / 'storage/seaweedfs/data').write_bytes(b'complete volume data') original = u.compose_strings(json.loads(subprocess.check_output([ 'docker', 'compose', '--project-directory', str(self.root), '--env-file', str(self.root / '.env'), '-f', str(self.root / 'docker-compose.yml'), 'config', '--format', 'json']))) u.write_json(self.job / 'original.json', original) target = copy.deepcopy(original) services = target['services'] services.pop('minio'); services.pop('createbuckets') services['db']['image'] = self.runner.db_image services['backend']['environment'].update(AWS_REQUEST_CHECKSUM_CALCULATION='WHEN_REQUIRED', FEDEO_RUN_MIGRATIONS='false') services['backend']['depends_on'] = {'db': {'condition': 'service_healthy'}, 'objectstore': {'condition': 'service_healthy'}} weed = u.read_json(ROOT / 'scripts/storage/seaweedfs.json') weed.update(image='sha256:' + 'e' * 64, container_name='fedeo-objectstore', restart='unless-stopped', networks={'internal': {'aliases': ['minio']}}, user='0:0', environment={'FEDEO_S3_CONFIG_SHA256': u.file_sha(self.root / 'storage/config/s3.json')}, volumes=[{'type': 'bind', 'source': str(self.root / 'storage/seaweedfs'), 'target': '/data'}, {'type': 'bind', 'source': str(self.root / 'storage/config'), 'target': '/etc/fedeo', 'read_only': True}]) services['objectstore'] = weed u.write_json(self.job / 'target.json', u.compose_strings(target, escape=True)) (self.root / 'docker-compose.yml').write_bytes((self.job / 'target.json').read_bytes()) network = original['networks']['internal']['name'] self.runner.storage = {'Id': 'objectstore', 'Image': weed['image'], 'Config': {'Labels': {'com.docker.compose.service': 'objectstore'}, 'Env': ['FEDEO_S3_CONFIG_SHA256=' + weed['environment']['FEDEO_S3_CONFIG_SHA256']]}, 'State': {'Running': True, 'Health': {'Status': 'healthy'}}, 'HostConfig': {}, 'NetworkSettings': {'Networks': {network: {'Aliases': ['minio']}}}, 'Mounts': [{'Type': 'bind', 'Source': v['source'], 'Destination': v['target'], 'RW': not v.get('read_only')} for v in weed['volumes']]} self.runner.old_minio = {'Id': 'retained-minio', 'Image': 'image-minio', 'Config': {'Labels': {'com.docker.compose.service': 'minio'}}, 'State': {'Running': False}, 'HostConfig': {'RestartPolicy': {'Name': 'no'}}, 'NetworkSettings': {'Networks': {}}} self.runner.local_images.update([weed['image'], self.runner.db_image]) state = {'schemaVersion': 1, 'phase': 'complete', 'provider': 'seaweedfs', 'writesReleased': True, 'version': '3.0.0', 'restoreVerifiedAt': u.now(), 'weedImage': weed['image'], 'network': network, 'fingerprint': {'.env': u.file_sha(self.root / '.env')}, 'count': 1, 'bytes': 20, 'containers': {name: {'id': name, 'image': 'image-' + name} for name in u.CORE_SERVICES | {'minio'}}} state['containers']['db']['image'] = self.runner.db_image state['containers']['minio']['id'] = 'retained-minio' u.write_json(self.job / 'copy/verified.json', {'count': 1, 'bytes': 20, 'verifiedAt': u.now(), 'objects': [{}]}) for name in ('restore-contract', 'acceptance', 'backend-acceptance'): u.write_json(self.job / name / 'result.json', {'operations': ['PUT', 'HEAD', 'GET', 'DELETE', 'SigV4', 'presigned-24h', 'anonymous-denied', 'wrong-key-denied']}) u.write_json(self.job / 'backup/complete.json', {}) (self.job / 'backup/seaweedfs.tar').write_bytes(b'verified cold restore archive') state['seaweedBackupSha256'] = u.file_sha(self.job / 'backup/seaweedfs.tar') u.write_json(self.job / 'state.json', state) u.write_json(self.root / '.updates/status.json', {'status': 'failed', 'phase': 'storage_migration', 'job': str(self.job), 'recoveryRequired': True}) return state def adopt(self): self.migrated() self.assertEqual(self.updater.adopt_storage()['status'], 'adopted') self.assertFalse(self.updater.status()['recoveryRequired']) self.runner.calls.clear() def test_adoption_is_read_only_for_services_and_idempotent(self): self.adopt() self.assertEqual(self.updater.adopt_storage()['status'], 'already_adopted') self.assertFalse(self.called('stop')) self.assertFalse(self.called('pull')) result = self.updater.execute('check', '3.1.0') self.assertEqual(result['plan']['storageProvider'], 'seaweedfs') self.assertFalse(result['plan']['requiresComposeAdoption']) self.assertEqual(result['plan']['storageBackupPaths'], ['storage']) def test_incomplete_migration_cannot_be_adopted_or_resolved(self): state = self.migrated() for fields in ({'phase': 'released'}, {'writesReleased': False}, {'restoreVerifiedAt': None}): u.write_json(self.job / 'state.json', {**state, **fields}) with self.assertRaisesRegex(u.UpdateError, 'vollständig'): self.updater.adopt_storage() self.assertTrue(self.updater.status()['recoveryRequired']) with self.assertRaisesRegex(u.UpdateError, 'adopt-storage'): self.updater.resolve('3.0.0') with self.assertRaisesRegex(u.UpdateError, 'adopt-storage'): self.run_update() def test_adoption_rejects_modified_backup_missing_contract_and_changed_environment(self): self.migrated() for path in (self.job / 'backup/seaweedfs.tar', self.job / 'backend-acceptance/result.json', self.root / '.env'): content = path.read_bytes() path.write_text('{}') with self.assertRaises(u.UpdateError): self.updater.adopt_storage() self.assertTrue(self.updater.status()['recoveryRequired']) path.write_bytes(content) self.assertFalse((self.root / '.updates/storage-provider.json').exists()) def test_update_retains_storage_and_database_and_backs_up_all_storage(self): self.adopt() self.runner.real_tar = True result = self.updater.execute('apply', '3.1.0') self.assertEqual(result['status'], 'succeeded') self.assertFalse(self.runner.old_minio['State']['Running']) self.assertTrue(self.runner.storage['State']['Running']) self.assertFalse(any('minio' in image or 'seaweed' in image or u.IMAGE_ID.fullmatch(image) for image in self.runner.downloads)) self.assertFalse(any('createbuckets' in a for a in self.runner.calls if 'run' in a)) backup = Path(result['backup']) with tarfile.open(backup / 'files.tar') as archive: self.assertEqual(archive.extractfile('storage/seaweedfs/data').read(), b'complete volume data') self.assertIn('storage/config/s3.json', archive.getnames()) self.assertNotIn('minio', archive.getnames()) self.assertTrue((backup / 'storage-image.tar').is_file()) self.assertTrue((backup / 'config/.updates/storage-provider.json').is_file()) calls = self.runner.calls contract = next(i for i, a in enumerate(calls) if u.S3_UPGRADE_CHECK in a) self.assertLess(contract, next(i for i, a in enumerate(calls) if 'migrate' in a)) # A later update also preserves the provider, with no new adoption required. (self.root / '.env').write_bytes(u.set_version((self.root / '.env').read_bytes(), '3.0.0')) target = u.read_json(self.root / 'docker-compose.yml') target['services']['backend']['environment']['FEDEO_VERSION'] = '3.0.0' u.write_json(self.root / 'docker-compose.yml', target) u.write_json(self.root / '.updates/installed.json', {'composeSha256': u.file_sha(self.root / 'docker-compose.yml')}) self.assertEqual(self.updater.execute('apply', '3.1.0')['status'], 'succeeded') def test_storage_drift_and_restarted_minio_block_before_maintenance(self): self.adopt() original = self.runner.old_minio['State'].copy() self.runner.old_minio['State']['Running'] = True with self.assertRaisesRegex(u.UpdateError, 'stillgelegt'): self.run_update() self.runner.old_minio['State'] = original (self.root / 'storage/config/s3.json').write_text('changed') with self.assertRaisesRegex(u.UpdateError, 'Konfiguration'): self.run_update() self.assertFalse(self.called('stop')) def test_storage_config_changed_during_pull_blocks_before_maintenance(self): self.adopt() self.runner.on_pull = lambda: (self.root / 'storage/config/s3.json').write_text('changed') with self.assertRaisesRegex(u.UpdateError, 'während'): self.run_update() self.assertFalse(self.called('stop')) def test_backup_failure_restarts_seaweed_without_restarting_minio(self): self.adopt() self.runner.fail = lambda a: a[0] == 'tar' and '-cpf' in a with self.assertRaises(u.UpdateError): self.run_update() self.assertFalse(self.updater.status()['recoveryRequired']) start = next(a for a in self.runner.calls if 'start' in a) self.assertIn('objectstore', start) self.assertNotIn('minio', start) def test_storage_must_remain_stopped_during_cold_backup(self): self.adopt() self.runner.on_stop = lambda: self.runner.storage['State'].update(Running=True) with self.assertRaisesRegex(u.UpdateError, 'kalten Sicherung'): self.run_update() self.assertFalse(self.called('migrate')) def test_target_s3_failure_blocks_database_migrations_and_retains_recovery(self): self.adopt() self.runner.fail = lambda a: u.S3_UPGRADE_CHECK in a with self.assertRaises(u.UpdateError): self.run_update() self.assertTrue(self.updater.status()['recoveryRequired']) self.assertFalse(self.called('migrate')) self.assertTrue(self.updater.status()['storageProbeKey'].startswith('.fedeo-update-check/')) self.runner.fail = None self.assertEqual(self.updater.resolve('3.1.0')['status'], 'recovered') self.assertTrue(self.called(u.S3_CLEANUP_CHECK)) self.assertIsNone(self.updater.status()['storageProbeKey']) def test_symlink_and_changed_database_major_are_rejected(self): self.adopt() link = self.root / 'storage/seaweedfs/external' link.symlink_to(self.root / 'minio', target_is_directory=True) with self.assertRaisesRegex(u.UpdateError, 'Links'): self.run_update() link.unlink() self.http.files['docker-compose.selfhost.yml'] = self.http.files['docker-compose.selfhost.yml'].replace(b'postgres:16', b'postgres:17') self.http.refresh() with self.assertRaisesRegex(u.UpdateError, 'PostgreSQL'): self.run_update() self.assertFalse(self.called('stop')) def test_literal_dollars_survive_generated_compose(self): self.adopt() value = 'secret-$FOO-$$-with-dollar' config = u.compose_strings(u.read_json(self.root / 'docker-compose.yml')) config['services']['backend']['environment']['COOKIE_SECRET'] = value u.write_json(self.root / 'docker-compose.yml', u.compose_strings(config, escape=True)) text = (self.root / '.env').read_text() import re (self.root / '.env').write_text(re.sub(r'(?m)^COOKIE_SECRET=.*$', "COOKIE_SECRET='" + value + "'", text)) result = self.run_update() self.assertEqual(result['status'], 'succeeded') self.assertEqual(self.updater.candidate['services']['backend']['environment']['COOKIE_SECRET'], value) def test_standard_installer_cannot_overwrite_a_planned_or_migrated_instance(self): import os original = (self.root / 'docker-compose.yml').read_bytes() (self.root / 'scripts').mkdir() setup = self.root / 'scripts/selfhost-setup.sh' setup.write_bytes((ROOT / 'scripts/selfhost-setup.sh').read_bytes()) for relative in ('storage', '.updates/storage-provider.json', '.updates/storage-migration/state.json'): path = self.root / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text('migration marker') result = subprocess.run(['bash', str(ROOT / 'scripts/selfhost-install.sh')], capture_output=True, text=True, env={**os.environ, 'FEDEO_DIR': str(self.root), 'FEDEO_TTY_INPUT': '/dev/null'}) self.assertEqual(result.returncode, 1) self.assertIn('nicht überschreiben', result.stderr) self.assertEqual((self.root / 'docker-compose.yml').read_bytes(), original) for args in (['--force', '--no-start'], ['--uninstall', '--purge']): result = subprocess.run(['bash', str(setup), *args], capture_output=True, text=True, env={**os.environ, 'FEDEO_TTY_INPUT': '/dev/null'}) self.assertEqual(result.returncode, 1) self.assertIn('nicht überschreiben oder entfernen', result.stderr) self.assertTrue((self.root / '.env').is_file()) path.unlink() if __name__ == '__main__': unittest.main()