Deploy MQTT test backend with private viewer and Gitea CI/CD

deploy/scripts/verify.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Verify the dedicated deployed broker using bounded clients; never print passwords."""
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+
+from prepare import IMAGE, USERS
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def invoke(args, timeout=30):
+    return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+
+
+def publication_result(result):
+    """Use the MQTT 5 PUBACK, not the CLI exit code, for broker acceptance."""
+    if result.returncode:
+        return result
+    acknowledgements = re.findall(r'received PUBACK \(Mid: \d+, RC:\s*(\d+)\)', result.stdout + result.stderr)
+    if len(acknowledgements) != 1:
+        raise RuntimeError('publish acknowledgement missing or ambiguous; check inconclusive')
+    reason = int(acknowledgements[0])
+    return subprocess.CompletedProcess(result.args, reason if reason >= 128 else 0,
+                                       result.stdout, result.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--insecure', action='store_true', help='also test the internal plaintext listener')
+    parser.add_argument('--restart', action='store_true', help='gracefully restart THIS test broker to check persistence')
+    args = parser.parse_args()
+    compose = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(ROOT / 'compose.yaml')]
+    if args.insecure:
+        compose += ['-f', str(ROOT / 'compose.insecure.yaml')]
+    try:
+        if invoke(compose + ['config', '--quiet']).returncode:
+            raise RuntimeError('Compose validation failed')
+        result = invoke(compose + ['ps', '-q', 'broker'])
+        container = result.stdout.strip()
+        if result.returncode or not container or len(container.splitlines()) != 1:
+            raise RuntimeError('one running test broker is required')
+        if not all(c in '0123456789abcdef' for c in container):
+            raise RuntimeError('invalid container ID')
+        check = invoke(compose + ['exec', '-T', 'broker', '/bin/sh', '/usr/local/bin/mqtt-healthcheck'])
+        if check.returncode:
+            raise RuntimeError('authenticated TLS health exchange failed')
+        print('Authenticated TLS publish/subscribe health check: PASS')
+        with tempfile.TemporaryDirectory(prefix='mqtt5-verify-') as folder:
+            temp = Path(folder)
+            for user in USERS:
+                if user == 'test-health':
+                    continue
+                record = json.loads((ROOT / 'secrets' / 'clients' / (user + '.json')).read_text())
+                password = record['password']
+                if record['username'] != user or not password or '\n' in password or '\r' in password:
+                    raise RuntimeError('invalid credential handoff file')
+                (temp / (user + '.conf')).write_text(f'-u {user}\n-P {password}\n')
+            (temp / 'anonymous.conf').write_text('')
+            (temp / 'wrong.conf').write_text('-u test-backend\n-P invalid-test-password\n')
+            for path in temp.iterdir():
+                path.chmod(0o600)
+            # A second unrelated test CA proves trust rejection; no existing CA is modified.
+            if invoke(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
+                       '-subj', '/CN=Untrusted TEST CA', '-keyout', str(temp / 'wrong.key'),
+                       '-out', str(temp / 'wrong.crt')]).returncode:
+                raise RuntimeError('unable to generate negative-test CA')
+            temp.joinpath('wrong.crt').chmod(0o644)
+
+            def client(binary, user, options, plaintext=False, host='localhost', wrong_ca=False):
+                base = ['docker', 'run', '--rm', '--network', 'container:' + container,
+                        '--user', f'{os.getuid()}:{os.getgid()}', '--read-only', '--cap-drop', 'ALL',
+                        '--security-opt', 'no-new-privileges',
+                        '--mount', f'type=bind,src={temp},dst=/checks,readonly',
+                        '--mount', f'type=bind,src={ROOT / "certs" / "server" / "ca.crt"},dst=/ca.crt,readonly',
+                        '--entrypoint', '/usr/bin/' + binary, IMAGE,
+                        '-o', '/checks/' + user + '.conf', '-h', host,
+                        '-p', '1883' if plaintext else '8883', '-V', 'mqttv5', '-q', '1']
+                base += ['--no-tls'] if plaintext else ['--cafile', '/checks/wrong.crt' if wrong_ca else '/ca.crt']
+                result = invoke(base + (['-d'] if binary == 'mosquitto_pub' else []) + options, timeout=15)
+                if result.returncode in (125, 126, 127):
+                    raise RuntimeError('Docker client container could not run; not a verified MQTT denial')
+                return publication_result(result) if binary == 'mosquitto_pub' else result
+
+            token = 'verify-' + uuid.uuid4().hex
+            topic = 'robot-market/test/v1/esp32-lab-001/status'
+            retained = False
+            session_id = 'rm5-check-' + uuid.uuid4().hex[:16]
+            try:
+                publish = ['-t', topic, '-r', '-m', token]
+                if client('mosquitto_pub', 'esp32-lab-001', publish).returncode:
+                    raise RuntimeError('authorized device publish failed')
+                retained = True
+                result = client('mosquitto_sub', 'test-backend', ['-t', topic, '-C', '1', '-W', '4'])
+                if result.returncode or result.stdout.strip() != token:
+                    raise RuntimeError('authorized backend receive failed')
+                print('Device publication and backend reception: PASS')
+                for name, user, kwargs in (
+                    ('Anonymous authentication', 'anonymous', {}),
+                    ('Invalid credentials', 'wrong', {}),
+                    ('Untrusted CA', 'test-backend', {'wrong_ca': True}),
+                    ('Wrong hostname', 'test-backend', {'host': '127.0.0.2'}),
+                ):
+                    result = client('mosquitto_sub', user, ['-t', topic, '-C', '1', '-W', '3'], **kwargs)
+                    if not result.returncode or result.stdout.strip() == token:
+                        raise RuntimeError(name + ' unexpectedly succeeded')
+                    # Denials must not simply be a subscriber timeout on a working connection.
+                    if result.returncode == 27:
+                        raise RuntimeError(name + ' was inconclusive (receive timeout)')
+                    print(name + ' rejection: PASS')
+                result = client('mosquitto_pub', 'sim-001', ['-t', topic, '-m', token])
+                if result.returncode:
+                    raise RuntimeError('functional cross-device publish failed')
+                print('Unrestricted cross-device publication: PASS')
+                result = client('mosquitto_sub', 'sim-001', ['-t', topic, '-C', '1', '-W', '3'])
+                if result.returncode or result.stdout.strip() != token:
+                    raise RuntimeError('functional cross-device read failed')
+                print('Unrestricted cross-device reception: PASS')
+                if args.insecure:
+                    result = client('mosquitto_sub', 'test-backend', ['-t', topic, '-C', '1', '-W', '3'], plaintext=True)
+                    if result.returncode or result.stdout.strip() != token:
+                        raise RuntimeError('plaintext authenticated exchange failed')
+                    print('Optional plaintext authenticated reception: PASS')
+                if args.restart:
+                    telemetry = 'robot-market/test/v1/sim-001/telemetry'
+                    result = client('mosquitto_sub', 'test-backend', ['-t', telemetry, '-c', '-x', '300', '-i', session_id, '-E'])
+                    if result.returncode:
+                        raise RuntimeError('persistent subscription setup failed')
+                    if invoke(compose + ['restart', 'broker'], timeout=45).returncode:
+                        raise RuntimeError('test broker restart failed')
+                    for attempt in range(10):
+                        if invoke(compose + ['exec', '-T', 'broker', '/bin/sh', '/usr/local/bin/mqtt-healthcheck']).returncode == 0:
+                            break
+                        time.sleep(1)
+                    else:
+                        raise RuntimeError('broker health did not recover')
+                    result = client('mosquitto_sub', 'test-backend', ['-t', topic, '-C', '1', '-W', '4'])
+                    if result.returncode or result.stdout.strip() != token:
+                        raise RuntimeError('retained status not preserved')
+                    if client('mosquitto_pub', 'sim-001', ['-t', telemetry, '-m', token]).returncode:
+                        raise RuntimeError('offline session queue publish failed')
+                    result = client('mosquitto_sub', 'test-backend', ['-t', telemetry, '-c', '-x', '0', '-i', session_id, '-C', '1', '-W', '4'])
+                    if result.returncode or result.stdout.strip() != token:
+                        raise RuntimeError('persistent subscription/queued message not restored')
+                    print('Graceful restart retained/session persistence: PASS (not crash durability)')
+            finally:
+                if retained:
+                    cleanup = client('mosquitto_pub', 'esp32-lab-001', ['-t', topic, '-r', '-n'])
+                    if cleanup.returncode:
+                        print('WARNING: verification retained status cleanup failed', file=sys.stderr)
+        print('Server-container checks complete. External DNS/firewall/hostname reachability still needs a client check.')
+        return 0
+    except (OSError, ValueError, KeyError, RuntimeError, subprocess.TimeoutExpired) as error:
+        # All RuntimeError messages above are controlled and contain no credential values.
+        message = str(error) if isinstance(error, RuntimeError) else 'missing prerequisites/files or a bounded operation failed'
+        print('Verification FAILED/BLOCKED: ' + message, file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())