Deploy MQTT test backend with private viewer and Gitea CI/CD

deploy/scripts/healthcheck.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Secrets are read from a protected config file, never process arguments.
+set -eu
+conf=/run/mqtt-secrets/health.conf
+topic=robot-market/test/health/check
+token="health-$(date +%s)-$$"
+cleanup() {
+    timeout 4 mosquitto_pub -o "$conf" -h localhost -p 8883 \
+        --cafile /mosquitto/certs/ca.crt -V mqttv5 -q 1 -t "$topic" -r -n \
+        >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+# Retain only for the subscribe check so startup scheduling cannot lose the probe.
+timeout 5 mosquitto_pub -o "$conf" -h localhost -p 8883 \
+    --cafile /mosquitto/certs/ca.crt -V mqttv5 -q 1 -t "$topic" -r -m "$token" \
+    >/dev/null 2>&1 || exit 1
+received=$(timeout 7 mosquitto_sub -o "$conf" -h localhost -p 8883 \
+    --cafile /mosquitto/certs/ca.crt -V mqttv5 -q 1 -t "$topic" -C 1 -W 5 \
+    2>/dev/null) || exit 1
+[ "$received" = "$token" ]

deploy/scripts/prepare.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Prepare test-only credentials/certificates without printing secrets or starting a broker."""
+import argparse
+import ipaddress
+import json
+import os
+from pathlib import Path
+import re
+import secrets
+import shutil
+import subprocess
+import sys
+
+IMAGE = 'eclipse-mosquitto:2.1.2-alpine@sha256:38c0da4f2ef84284d47b3b3eeea1cb3bdeabe81ee10caf0cd5c5ff61ee3ea408'
+USERS = ('test-backend', 'sim-001', 'esp32-lab-001', 'test-health')
+
+
+def run(args):
+    result = subprocess.run(args, capture_output=True, text=True, timeout=180)
+    if result.returncode:
+        # OpenSSL/CLI errors can contain paths or config values. Keep raw output private.
+        raise RuntimeError(f'{args[0]} failed; check tool availability, Docker access and image availability')
+
+
+def valid_host(value):
+    try:
+        ipaddress.ip_address(value)
+        return value
+    except ValueError:
+        if len(value) > 253 or not all(re.fullmatch(r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?', part) for part in value.split('.')):
+            raise argparse.ArgumentTypeError('expected a DNS hostname or IP address, without scheme/port')
+        return value
+
+
+def write(path, data, mode=0o600):
+    with path.open('x') as stream:
+        stream.write(data)
+    path.chmod(mode)
+
+
+def prepare(root, host, uid, gid):
+    root.mkdir(parents=True, exist_ok=True)
+    for name in ('certs', 'secrets', 'data', '.env'):
+        if (root / name).exists() or (root / name).is_symlink():
+            raise RuntimeError(f'{name} already exists; refusing to replace existing credentials, certificates or data')
+    if os.geteuid() != 0 and (uid != os.getuid() or gid != os.getgid()):
+        raise RuntimeError('run with sudo for UID/GID 1883, or supply --owner with your own UID:GID for isolated artifact tests')
+    for tool in ('docker', 'openssl'):
+        if not shutil.which(tool):
+            raise RuntimeError(f'{tool} is required')
+    # All following generated content is inside the explicitly selected deployment root.
+    os.umask(0o077)
+    authority = root / 'certs' / 'authority'
+    server = root / 'certs' / 'server'
+    runtime = root / 'secrets' / 'runtime'
+    clients = root / 'secrets' / 'clients'
+    for path in (authority, server, runtime, clients, root / 'data'):
+        path.mkdir(parents=True)
+    run(['openssl', 'req', '-x509', '-newkey', 'rsa:3072', '-nodes', '-sha256', '-days', '365',
+         '-subj', '/CN=Robot Market TEST CA', '-addext', 'basicConstraints=critical,CA:TRUE',
+         '-addext', 'keyUsage=critical,keyCertSign,cRLSign',
+         '-keyout', str(authority / 'ca.key'), '-out', str(server / 'ca.crt')])
+    try:
+        ipaddress.ip_address(host)
+        host_san = f'IP:{host}'
+    except ValueError:
+        host_san = f'DNS:{host}'
+    write(server / 'server.ext', 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=' + host_san + ',DNS:localhost,IP:127.0.0.1\n')
+    run(['openssl', 'req', '-new', '-newkey', 'rsa:3072', '-nodes', '-sha256', '-subj', '/CN=' + host,
+         '-keyout', str(server / 'server.key'), '-out', str(server / 'server.csr')])
+    run(['openssl', 'x509', '-req', '-sha256', '-days', '90', '-in', str(server / 'server.csr'),
+         '-CA', str(server / 'ca.crt'), '-CAkey', str(authority / 'ca.key'), '-CAcreateserial',
+         '-extfile', str(server / 'server.ext'), '-out', str(server / 'server.crt')])
+    run(['openssl', 'verify', '-CAfile', str(server / 'ca.crt'), str(server / 'server.crt')])
+    passwords = {user: secrets.token_urlsafe(32) for user in USERS}
+    # Hash a disposable file; never pass passwords as command arguments.
+    raw = runtime / 'passwords.pending'
+    write(raw, ''.join(f'{user}:{password}\n' for user, password in passwords.items()))
+    try:
+        run(['docker', 'run', '--rm', '--network', 'none', '--user', f'{os.getuid()}:{os.getgid()}',
+             '--mount', f'type=bind,src={runtime.resolve()},dst=/setup',
+             '--entrypoint', '/usr/bin/mosquitto_passwd', IMAGE, '-U', '/setup/passwords.pending'])
+        # Check conversion without emitting hashes or passwords.
+        entries = raw.read_text().splitlines()
+        if len(entries) != len(USERS) or any(not line.split(':', 1)[1].startswith('$') for line in entries):
+            raise RuntimeError('password hashing did not produce the expected output')
+        raw.rename(runtime / 'passwords')
+    finally:
+        raw.unlink(missing_ok=True)
+    write(runtime / 'health.conf', '-u test-health\n-P ' + passwords['test-health'] + '\n')
+    for user in USERS:
+        if user == 'test-health':
+            continue
+        write(clients / (user + '.json'), json.dumps({'host': host, 'tls_port': 8884, 'plaintext_port': 1884,
+              'username': user, 'password': passwords[user], 'device_id': user if user != 'test-backend' else None}, indent=2) + '\n')
+    for path in (root / 'certs', server, root / 'secrets', runtime):
+        os.chown(path, os.getuid(), gid)
+        path.chmod(0o750)
+    for path in (*server.iterdir(), *runtime.iterdir()):
+        os.chown(path, os.getuid(), gid)
+        path.chmod(0o640)
+    # The CA signing key stays private and is never mounted into the broker.
+    authority.chmod(0o700)
+    (authority / 'ca.key').chmod(0o600)
+    os.chown(root / 'data', uid, gid)
+    (root / 'data').chmod(0o750)
+    write(root / '.env', f'TLS_BIND_IP=0.0.0.0\nTLS_PORT=8884\nINSECURE_BIND_IP=0.0.0.0\nINSECURE_PORT=1884\nBROKER_UID={uid}\nBROKER_GID={gid}\n', 0o644)
+    print('Prepared test certificates, hashed broker passwords and private client handoff files.')
+    print('No broker was started. Edit .env for available ports; client JSON ports must match your edits.')
+    print('Client secrets: secrets/clients/ (do not commit or paste into public logs). Public CA: certs/server/ca.crt')
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--host', required=True, type=valid_host, help='actual DNS name or IP clients will use')
+    parser.add_argument('--owner', default='1883:1883', help='broker UID:GID (default 1883:1883)')
+    parser.add_argument('--output', type=Path, default=Path(__file__).resolve().parents[1])
+    args = parser.parse_args()
+    try:
+        uid, gid = map(int, args.owner.split(':'))
+        if not (1 <= uid <= 2**31-1 and 1 <= gid <= 2**31-1):
+            raise ValueError()
+    except ValueError:
+        parser.error('--owner must be non-root numeric UID:GID')
+    try:
+        prepare(args.output.resolve(), args.host, uid, gid)
+    except (OSError, RuntimeError, subprocess.TimeoutExpired) as error:
+        detail = str(error) if isinstance(error, RuntimeError) else 'a file operation or bounded tool execution failed'
+        print('Preparation failed: ' + detail + '. Preserve partial output and inspect privately before retrying.', file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

deploy/scripts/verify.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Verify the dedicated deployed broker using bounded clients; never print passwords."""
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+
+from prepare import IMAGE, USERS
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def invoke(args, timeout=30):
+    return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+
+
+def publication_result(result):
+    """Use the MQTT 5 PUBACK, not the CLI exit code, for broker acceptance."""
+    if result.returncode:
+        return result
+    acknowledgements = re.findall(r'received PUBACK \(Mid: \d+, RC:\s*(\d+)\)', result.stdout + result.stderr)
+    if len(acknowledgements) != 1:
+        raise RuntimeError('publish acknowledgement missing or ambiguous; check inconclusive')
+    reason = int(acknowledgements[0])
+    return subprocess.CompletedProcess(result.args, reason if reason >= 128 else 0,
+                                       result.stdout, result.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--insecure', action='store_true', help='also test the internal plaintext listener')
+    parser.add_argument('--restart', action='store_true', help='gracefully restart THIS test broker to check persistence')
+    args = parser.parse_args()
+    compose = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(ROOT / 'compose.yaml')]
+    if args.insecure:
+        compose += ['-f', str(ROOT / 'compose.insecure.yaml')]
+    try:
+        if invoke(compose + ['config', '--quiet']).returncode:
+            raise RuntimeError('Compose validation failed')
+        result = invoke(compose + ['ps', '-q', 'broker'])
+        container = result.stdout.strip()
+        if result.returncode or not container or len(container.splitlines()) != 1:
+            raise RuntimeError('one running test broker is required')
+        if not all(c in '0123456789abcdef' for c in container):
+            raise RuntimeError('invalid container ID')
+        check = invoke(compose + ['exec', '-T', 'broker', '/bin/sh', '/usr/local/bin/mqtt-healthcheck'])
+        if check.returncode:
+            raise RuntimeError('authenticated TLS health exchange failed')
+        print('Authenticated TLS publish/subscribe health check: PASS')
+        with tempfile.TemporaryDirectory(prefix='mqtt5-verify-') as folder:
+            temp = Path(folder)
+            for user in USERS:
+                if user == 'test-health':
+                    continue
+                record = json.loads((ROOT / 'secrets' / 'clients' / (user + '.json')).read_text())
+                password = record['password']
+                if record['username'] != user or not password or '\n' in password or '\r' in password:
+                    raise RuntimeError('invalid credential handoff file')
+                (temp / (user + '.conf')).write_text(f'-u {user}\n-P {password}\n')
+            (temp / 'anonymous.conf').write_text('')
+            (temp / 'wrong.conf').write_text('-u test-backend\n-P invalid-test-password\n')
+            for path in temp.iterdir():
+                path.chmod(0o600)
+            # A second unrelated test CA proves trust rejection; no existing CA is modified.
+            if invoke(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
+                       '-subj', '/CN=Untrusted TEST CA', '-keyout', str(temp / 'wrong.key'),
+                       '-out', str(temp / 'wrong.crt')]).returncode:
+                raise RuntimeError('unable to generate negative-test CA')
+            temp.joinpath('wrong.crt').chmod(0o644)
+
+            def client(binary, user, options, plaintext=False, host='localhost', wrong_ca=False):
+                base = ['docker', 'run', '--rm', '--network', 'container:' + container,
+                        '--user', f'{os.getuid()}:{os.getgid()}', '--read-only', '--cap-drop', 'ALL',
+                        '--security-opt', 'no-new-privileges',
+                        '--mount', f'type=bind,src={temp},dst=/checks,readonly',
+                        '--mount', f'type=bind,src={ROOT / "certs" / "server" / "ca.crt"},dst=/ca.crt,readonly',
+                        '--entrypoint', '/usr/bin/' + binary, IMAGE,
+                        '-o', '/checks/' + user + '.conf', '-h', host,
+                        '-p', '1883' if plaintext else '8883', '-V', 'mqttv5', '-q', '1']
+                base += ['--no-tls'] if plaintext else ['--cafile', '/checks/wrong.crt' if wrong_ca else '/ca.crt']
+                result = invoke(base + (['-d'] if binary == 'mosquitto_pub' else []) + options, timeout=15)
+                if result.returncode in (125, 126, 127):
+                    raise RuntimeError('Docker client container could not run; not a verified MQTT denial')
+                return publication_result(result) if binary == 'mosquitto_pub' else result
+
+            token = 'verify-' + uuid.uuid4().hex
+            topic = 'robot-market/test/v1/esp32-lab-001/status'
+            retained = False
+            session_id = 'rm5-check-' + uuid.uuid4().hex[:16]
+            try:
+                publish = ['-t', topic, '-r', '-m', token]
+                if client('mosquitto_pub', 'esp32-lab-001', publish).returncode:
+                    raise RuntimeError('authorized device publish failed')
+                retained = True
+                result = client('mosquitto_sub', 'test-backend', ['-t', topic, '-C', '1', '-W', '4'])
+                if result.returncode or result.stdout.strip() != token:
+                    raise RuntimeError('authorized backend receive failed')
+                print('Device publication and backend reception: PASS')
+                for name, user, kwargs in (
+                    ('Anonymous authentication', 'anonymous', {}),
+                    ('Invalid credentials', 'wrong', {}),
+                    ('Untrusted CA', 'test-backend', {'wrong_ca': True}),
+                    ('Wrong hostname', 'test-backend', {'host': '127.0.0.2'}),
+                ):
+                    result = client('mosquitto_sub', user, ['-t', topic, '-C', '1', '-W', '3'], **kwargs)
+                    if not result.returncode or result.stdout.strip() == token:
+                        raise RuntimeError(name + ' unexpectedly succeeded')
+                    # Denials must not simply be a subscriber timeout on a working connection.
+                    if result.returncode == 27:
+                        raise RuntimeError(name + ' was inconclusive (receive timeout)')
+                    print(name + ' rejection: PASS')
+                result = client('mosquitto_pub', 'sim-001', ['-t', topic, '-m', token])
+                if result.returncode:
+                    raise RuntimeError('functional cross-device publish failed')
+                print('Unrestricted cross-device publication: PASS')
+                result = client('mosquitto_sub', 'sim-001', ['-t', topic, '-C', '1', '-W', '3'])
+                if result.returncode or result.stdout.strip() != token:
+                    raise RuntimeError('functional cross-device read failed')
+                print('Unrestricted cross-device reception: PASS')
+                if args.insecure:
+                    result = client('mosquitto_sub', 'test-backend', ['-t', topic, '-C', '1', '-W', '3'], plaintext=True)
+                    if result.returncode or result.stdout.strip() != token:
+                        raise RuntimeError('plaintext authenticated exchange failed')
+                    print('Optional plaintext authenticated reception: PASS')
+                if args.restart:
+                    telemetry = 'robot-market/test/v1/sim-001/telemetry'
+                    result = client('mosquitto_sub', 'test-backend', ['-t', telemetry, '-c', '-x', '300', '-i', session_id, '-E'])
+                    if result.returncode:
+                        raise RuntimeError('persistent subscription setup failed')
+                    if invoke(compose + ['restart', 'broker'], timeout=45).returncode:
+                        raise RuntimeError('test broker restart failed')
+                    for attempt in range(10):
+                        if invoke(compose + ['exec', '-T', 'broker', '/bin/sh', '/usr/local/bin/mqtt-healthcheck']).returncode == 0:
+                            break
+                        time.sleep(1)
+                    else:
+                        raise RuntimeError('broker health did not recover')
+                    result = client('mosquitto_sub', 'test-backend', ['-t', topic, '-C', '1', '-W', '4'])
+                    if result.returncode or result.stdout.strip() != token:
+                        raise RuntimeError('retained status not preserved')
+                    if client('mosquitto_pub', 'sim-001', ['-t', telemetry, '-m', token]).returncode:
+                        raise RuntimeError('offline session queue publish failed')
+                    result = client('mosquitto_sub', 'test-backend', ['-t', telemetry, '-c', '-x', '0', '-i', session_id, '-C', '1', '-W', '4'])
+                    if result.returncode or result.stdout.strip() != token:
+                        raise RuntimeError('persistent subscription/queued message not restored')
+                    print('Graceful restart retained/session persistence: PASS (not crash durability)')
+            finally:
+                if retained:
+                    cleanup = client('mosquitto_pub', 'esp32-lab-001', ['-t', topic, '-r', '-n'])
+                    if cleanup.returncode:
+                        print('WARNING: verification retained status cleanup failed', file=sys.stderr)
+        print('Server-container checks complete. External DNS/firewall/hostname reachability still needs a client check.')
+        return 0
+    except (OSError, ValueError, KeyError, RuntimeError, subprocess.TimeoutExpired) as error:
+        # All RuntimeError messages above are controlled and contain no credential values.
+        message = str(error) if isinstance(error, RuntimeError) else 'missing prerequisites/files or a bounded operation failed'
+        print('Verification FAILED/BLOCKED: ' + message, file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())