feat(selfhost): show sanitized update logs in admin UI

scripts/update/tests/test_service.py

@@ -8,6 +8,7 @@ import subprocess
 import tempfile
 import threading
 import unittest
+from unittest.mock import patch
 
 ROOT = Path(__file__).resolve().parents[3]
 spec = importlib.util.spec_from_file_location('update_service', ROOT / 'scripts/selfhost-update-service.py')
@@ -232,6 +233,86 @@ class ServiceTests(unittest.TestCase):
         self.assertEqual(self.request('GET', '/v1/logs')[0], 404)
         self.assertEqual((self.controller.home / 'check-request-01.log').stat().st_mode & 0o777, 0o600)
 
+    def test_logs_include_failure_commands_and_events_with_credentials_redacted(self):
+        (self.root / '.env').write_text('POSTGRES_PASSWORD="database-secret"\nAPI_TOKEN=api-secret\n'
+                                      'SMTP_PASSWORD=complex,secret/suffix\n'
+                                      'DATABASE_URL=postgres://user:db%2Fsecret@db/fedeo\n')
+        (self.root / 'docker-compose.override.yml').write_text('  MINIO_ROOT_PASSWORD: "minio-secret"\n')
+        self.factory.failure = s.engine.UpdateError('Registry unauthorized: database-secret')
+        request_id = self.check()
+        job = self.root / '.updates' / ('job-' + request_id)
+        (job / 'target.env').write_text("SMTP_PASSWORD='previous-secret'\n")
+        (job / 'commands.log').write_text(
+            '\x1b[31mError: pull access denied\x1b[0m\n'
+            'database-secret api-secret minio-secret previous-secret release-secret\n'
+            'complex,secret/suffix complex%2Csecret%2Fsuffix db/secret\n'
+            'https://user:url-secret@registry.test/image\n'
+            'Authorization: Bearer header-secret\n'
+            '{"password": "json-secret", "access_token": "query-secret"}\n'
+            'TOKEN=unquoted-secret\n--password cli-secret\n')
+        with patch.dict(s.os.environ, {'FEDEO_RELEASE_TOKEN': 'release-secret'}):
+            status, result = self.request('GET', f'/v1/requests/{request_id}/logs')
+        self.assertEqual(status, 200)
+        self.assertEqual(result['requestId'], request_id)
+        logs = {entry['source']: entry['content'] for entry in result['logs']}
+        self.assertIn('Registry unauthorized', logs['service'])
+        self.assertIn('pull access denied', logs['commands'])
+        self.assertIn('validating', logs['events'])
+        for secret in ('database-secret', 'api-secret', 'minio-secret', 'previous-secret', 'release-secret',
+                       'url-secret', 'header-secret', 'json-secret', 'query-secret', 'unquoted-secret', 'cli-secret',
+                       'complex,secret/suffix', 'complex%2Csecret%2Fsuffix', 'db/secret', '\x1b'):
+            self.assertNotIn(secret, json.dumps(result))
+        self.assertTrue(self.request('GET', '/v1/status')[1]['capabilities']['requestLogs'])
+
+    def test_logs_survive_restart_and_find_legacy_jobs_without_crossing_requests(self):
+        first = self.check()
+        first_job = self.root / '.updates' / ('job-' + first)
+        (first_job / 'commands.log').write_text('first request only\n')
+        # Simulate the private request record from an older host service.
+        record = self.controller.read(first)
+        record.pop('job')
+        s.engine.write_json(self.controller.path(first), record)
+        second = self.check('check-request-02')
+        (self.root / '.updates' / ('job-' + second) / 'commands.log').write_text('second request only\n')
+        self.controller.close()
+        self.controller = s.Controller(self.root, updater_factory=self.factory)
+        self.server.controller = self.controller
+        for request_id, own, other in [(first, 'first', 'second'), (second, 'second', 'first')]:
+            payload = json.dumps(self.request('GET', f'/v1/requests/{request_id}/logs')[1])
+            self.assertIn(own + ' request only', payload)
+            self.assertNotIn(other + ' request only', payload)
+        self.assertEqual(self.request('GET', '/v1/requests/missing-request/logs')[0], 404)
+        self.assertEqual(self.request('GET', '/v1/requests/../../secret/logs')[0], 400)
+
+    def test_running_logs_are_bounded_and_do_not_expose_partial_secret_lines(self):
+        self.factory.release.clear()
+        request_id = 'check-running-logs'
+        self.request('POST', '/v1/check', {'requestId': request_id})
+        self.assertTrue(self.factory.started.wait(3))
+        job = self.root / '.updates' / ('job-' + request_id)
+        (job / 'commands.log').write_text('PASSWORD=' + 'secret-fragment' * 4000 + '\n' + 'old entry\n' * 900 + 'Last diagnostic: ä\n')
+        result = self.request('GET', f'/v1/requests/{request_id}/logs')[1]
+        commands = next(entry for entry in result['logs'] if entry['source'] == 'commands')
+        self.assertTrue(commands['truncated'])
+        self.assertLessEqual(len(commands['content'].encode()), s.LOG_BYTES)
+        self.assertIn('Last diagnostic: ä', commands['content'])
+        self.assertNotIn('secret-fragment', commands['content'])
+        self.assertLess(len(json.dumps(result).encode()), 65536)
+
+    def test_logs_fail_closed_for_symlinks_and_unreadable_redaction_configuration(self):
+        request_id = self.check()
+        job = self.root / '.updates' / ('job-' + request_id)
+        private = self.root / 'private.txt'
+        private.write_text('never publish this')
+        (job / 'commands.log').symlink_to(private)
+        status, result = self.request('GET', f'/v1/requests/{request_id}/logs')
+        self.assertEqual(status, 503)
+        self.assertEqual(result['error']['code'], 'LOGS_UNAVAILABLE')
+        self.assertNotIn('never publish this', json.dumps(result))
+        (job / 'commands.log').unlink()
+        (self.root / '.env').symlink_to(private)
+        self.assertEqual(self.request('GET', f'/v1/requests/{request_id}/logs')[0], 503)
+
     def test_api_rejects_unsupported_commands_options_bodies_and_browser_requests(self):
         for body in [[], {'requestId': 'bad/path'}, {'requestId': 'check-request-01', 'version': ';reboot'},
                      {'requestId': 'check-request-01', 'directory': '/etc'},