Deploy MQTT test backend with private viewer and Gitea CI/CD

internal/integration/broker_test.go

@@ -0,0 +1,244 @@
+package integration
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"robot-market/test-mqtt-5/internal/backend"
+	"robot-market/test-mqtt-5/internal/config"
+	"robot-market/test-mqtt-5/internal/contracts"
+	"robot-market/test-mqtt-5/internal/events"
+	"robot-market/test-mqtt-5/internal/mqtt"
+	"robot-market/test-mqtt-5/internal/simulator"
+)
+
+func remoteConfig(t *testing.T, role string, plaintext bool) config.Config {
+	t.Helper()
+	root := os.Getenv("MQTT5_INTEGRATION_CREDENTIAL_DIR")
+	if root == "" {
+		t.Skip("opt-in remote test: set MQTT5_INTEGRATION_CREDENTIAL_DIR")
+	}
+	data, err := os.ReadFile(filepath.Join(root, role+".json"))
+	if err != nil {
+		t.Fatal("cannot read private credential handoff")
+	}
+	var record struct {
+		Host     string `json:"host"`
+		TLS      int    `json:"tls_port"`
+		Plain    int    `json:"plaintext_port"`
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if json.Unmarshal(data, &record) != nil {
+		t.Fatal("invalid credential handoff")
+	}
+	cfg := config.Defaults()
+	cfg.MQTT.Username = record.Username
+	cfg.MQTT.Password = record.Password
+	cfg.MQTT.ClientID = "rm5-integration-" + backend.ID()[:16]
+	scheme, port := "tls", record.TLS
+	if plaintext {
+		scheme, port = "mqtt", record.Plain
+		cfg.MQTT.AllowPlaintext = true
+	}
+	cfg.MQTT.URL = scheme + "://" + record.Host + ":" + jsonNumber(port)
+	cfg.TLS.CAFile = filepath.Join(root, "ca.crt")
+	cfg.Simulator.TelemetryInterval = 200 * time.Millisecond
+	if err := cfg.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	reportScenario(t, cfg)
+	return cfg
+}
+
+var registeredResults sync.Map
+
+func reportScenario(t *testing.T, cfg config.Config) {
+	if _, loaded := registeredResults.LoadOrStore(t, true); loaded {
+		return
+	}
+	t.Cleanup(func() {
+		registeredResults.Delete(t)
+		status := "passed"
+		if t.Failed() {
+			status = "failed"
+		}
+		if t.Skipped() {
+			status = "blocked"
+		}
+		result := events.ScenarioResult{Feature: t.Name(), Environment: "independent Go integration clients and remote test broker; no ESP32 hardware", Endpoint: cfg.MQTT.URL, Version: "MQTT 5 / Mosquitto 2.1.2", Status: status, Observation: "See named Go test assertions and verbose output; application duplicates are separate from wire retransmission", Time: time.Now().UTC()}
+		data, _ := json.Marshal(result)
+		t.Log("scenario_result " + string(data))
+		ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
+		defer cancel()
+		cfg.MQTT.ClientID = "rm5-result-" + backend.ID()[:12]
+		cfg.MQTT.SessionExpiry = 0
+		cfg.Topics.ObserveAll = true
+		client := mqtt.New(cfg, store(cfg))
+		if err := client.Start(ctx, nil, nil); err != nil {
+			t.Error("scenario reporting connection failed")
+			return
+		}
+		defer client.Close()
+		if err := client.PublishRaw(ctx, mqtt.Packet(cfg.Topics.Namespace+"/scenarios/results", 1, false, result)); err != nil {
+			t.Error("scenario reporting publication failed")
+		}
+	})
+}
+func jsonNumber(n int) string { data, _ := json.Marshal(n); return string(data) }
+func store(cfg config.Config) *events.Store {
+	return events.NewStore(cfg.Limits.EventHistory, slog.New(slog.NewTextHandler(io.Discard, nil)))
+}
+func wait(t *testing.T, f func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for !f() {
+		if time.Now().After(deadline) {
+			t.Fatal("condition deadline exceeded")
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+func TestRemoteLoop(t *testing.T) {
+	for _, plain := range []bool{false, true} {
+		name := "TLS"
+		if plain {
+			name = "Plaintext"
+		}
+		t.Run(name, func(t *testing.T) {
+			cfg := remoteConfig(t, "sim-001", plain)
+			simClient := mqtt.New(cfg, store(cfg))
+			sim := simulator.New(simClient)
+			ctx, cancel := context.WithCancel(context.Background())
+			simDone := make(chan error, 1)
+			go func() { simDone <- sim.Run(ctx) }()
+			defer func() {
+				start := time.Now()
+				cancel()
+				select {
+				case err := <-simDone:
+					if err != nil {
+						t.Error(err)
+					}
+				case <-time.After(6 * time.Second):
+					t.Error("simulator shutdown blocked")
+				}
+				if time.Since(start) > cfg.Deadlines.Shutdown {
+					t.Error("simulator exceeded configured shutdown bound")
+				}
+			}()
+			wait(t, simClient.Ready)
+			backendCfg := remoteConfig(t, "test-backend", plain)
+			backendClient := mqtt.New(backendCfg, store(backendCfg))
+			if err := backendClient.Start(ctx, []string{"telemetry", "status", "commands", "acks"}, nil); err != nil {
+				t.Fatal(err)
+			}
+			defer backendClient.Close()
+			b := backend.New(backendClient)
+			workerCtx, stop := context.WithCancel(ctx)
+			done := make(chan error, 1)
+			go func() { done <- b.Serve(workerCtx) }()
+			defer func() {
+				stop()
+				if err := <-done; err != nil {
+					t.Error(err)
+				}
+			}()
+			wait(t, func() bool {
+				history, _ := backendClient.Events.Snapshot()
+				seenTelemetry, seenStatus := false, false
+				for _, event := range history {
+					if event.Kind == events.MessageReceived && event.Topic == mqtt.Topic(backendCfg, "telemetry") {
+						seenTelemetry = true
+					}
+					if event.Kind == events.MessageReceived && event.Topic == mqtt.Topic(backendCfg, "status") && strings.Contains(event.Payload.Text, `"state":"online"`) {
+						seenStatus = true
+					}
+				}
+				return seenTelemetry && seenStatus
+			})
+			command := backend.NewCommand("sim-001", "ping", backend.ID(), 5*time.Second)
+			for n := 0; n < 2; n++ {
+				ack, err := b.Command(ctx, command)
+				if err != nil || ack == nil || ack.Result.Pong == nil || !*ack.Result.Pong {
+					t.Fatal("ping/replay did not complete")
+				}
+			}
+			_, counts := simClient.Events.Snapshot()
+			if counts[events.Kind("simulator_execution")] != 1 || counts[events.DuplicateRequest] < 1 {
+				t.Fatal("duplicate caused re-execution or replay missing")
+			}
+			conflict := command
+			conflict.Command = "get_version"
+			if _, err := b.Command(ctx, conflict); err == nil {
+				t.Fatal("conflicting ID was not rejected")
+			}
+			ack, err := b.Command(ctx, backend.NewCommand("sim-001", "get_version", backend.ID(), 5*time.Second))
+			if err != nil || ack.Result.Version != "test-sim-1.0.0" {
+				t.Fatal("version result failed")
+			}
+			t.Log("observer telemetry/status, native correlated ping/version, cached duplicate and conflict rejection: PASS")
+		})
+	}
+}
+func TestRemoteWrongCorrelationTimeoutAndDisconnect(t *testing.T) {
+	cfg := remoteConfig(t, "sim-001", false)
+	cfg.Simulator.AckDelay = 2 * time.Second
+	simClient := mqtt.New(cfg, store(cfg))
+	sim := simulator.New(simClient)
+	ctx, cancel := context.WithCancel(context.Background())
+	simDone := make(chan error, 1)
+	go func() { simDone <- sim.Run(ctx) }()
+	defer func() {
+		cancel()
+		select {
+		case <-simDone:
+		case <-time.After(6 * time.Second):
+			t.Error("sim shutdown blocked")
+		}
+	}()
+	wait(t, simClient.Ready)
+	bc := remoteConfig(t, "test-backend", false)
+	client := mqtt.New(bc, store(bc))
+	if err := client.Start(ctx, []string{"acks"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	b := backend.New(client)
+	workerCtx, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- b.Serve(workerCtx) }()
+	defer func() { stop(); <-done }()
+	command := backend.NewCommand("sim-001", "ping", backend.ID(), time.Second)
+	commandDone := make(chan error, 1)
+	go func() { _, err := b.Command(ctx, command); commandDone <- err }()
+	wait(t, func() bool { _, counts := client.Events.Snapshot(); return counts[events.PublishAttempt] >= 1 })
+	yes := true
+	badAck := contracts.Ack{Envelope: command.Envelope, RequestID: command.RequestID, Timestamp: backend.Timestamp(time.Now()), Stage: "completed", OK: &yes, Result: &contracts.Result{Pong: &yes}}
+	packet := mqtt.Packet(mqtt.Topic(cfg, "acks"), 1, false, badAck)
+	packet.Properties.CorrelationData = []byte("wrong-correlation")
+	if err := simClient.Publish(ctx, packet); err != nil {
+		t.Fatal("invalid-correlation test publication failed")
+	}
+	if err := <-commandDone; err == nil || !strings.Contains(err.Error(), "deadline") {
+		t.Fatal("invalid correlation completed request or truthful timeout missing")
+	}
+	wait(t, func() bool { _, counts := client.Events.Snapshot(); return counts[events.LateAck] > 0 })
+	command = backend.NewCommand("sim-001", "ping", backend.ID(), 5*time.Second)
+	go func() { _, err := b.Command(ctx, command); commandDone <- err }()
+	wait(t, func() bool { _, counts := client.Events.Snapshot(); return counts[events.PublishAttempt] >= 2 })
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := <-commandDone; err == nil {
+		t.Fatal("disconnect claimed execution success")
+	}
+	t.Log("wrong correlation ignored, missing valid completion timeout, late ACK classification and disconnect unknown outcome: PASS")
+}

internal/integration/reconnect_test.go

@@ -0,0 +1,172 @@
+package integration
+
+import (
+	"context"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"robot-market/test-mqtt-5/internal/backend"
+	"robot-market/test-mqtt-5/internal/events"
+	"robot-market/test-mqtt-5/internal/mqtt"
+	"robot-market/test-mqtt-5/internal/simulator"
+)
+
+// A loopback byte relay drops only this test client's connection. No broker changes.
+type relay struct {
+	listener    net.Listener
+	target      string
+	mu          sync.Mutex
+	paused      bool
+	connections map[net.Conn]bool
+	wg          sync.WaitGroup
+}
+
+func newRelay(t *testing.T, target string) *relay {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := &relay{listener: listener, target: target, connections: make(map[net.Conn]bool)}
+	r.wg.Add(1)
+	go func() {
+		defer r.wg.Done()
+		for {
+			local, err := listener.Accept()
+			if err != nil {
+				return
+			}
+			r.mu.Lock()
+			if r.paused {
+				r.mu.Unlock()
+				local.Close()
+				continue
+			}
+			r.connections[local] = true
+			r.wg.Add(1)
+			r.mu.Unlock()
+			go r.forward(local)
+		}
+	}()
+	return r
+}
+func (r *relay) forward(local net.Conn) {
+	defer r.wg.Done()
+	defer local.Close()
+	defer func() { r.mu.Lock(); delete(r.connections, local); r.mu.Unlock() }()
+	remote, err := net.DialTimeout("tcp", r.target, 3*time.Second)
+	if err != nil {
+		return
+	}
+	defer remote.Close()
+	r.mu.Lock()
+	if r.paused {
+		r.mu.Unlock()
+		return
+	}
+	r.connections[remote] = true
+	r.mu.Unlock()
+	defer func() { r.mu.Lock(); delete(r.connections, remote); r.mu.Unlock() }()
+	done := make(chan struct{})
+	go func() { io.Copy(remote, local); remote.Close(); close(done) }()
+	io.Copy(local, remote)
+	local.Close()
+	<-done
+}
+func (r *relay) pause() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.paused = true
+	for c := range r.connections {
+		c.Close()
+	}
+}
+func (r *relay) resume() { r.mu.Lock(); r.paused = false; r.mu.Unlock() }
+func (r *relay) close()  { r.listener.Close(); r.pause(); r.wg.Wait() }
+func TestRemoteReconnectSessionAndUnknown(t *testing.T) {
+	for _, resumed := range []bool{true, false} {
+		name := "SessionResumed"
+		if !resumed {
+			name = "SessionLostSubscriptionsRestored"
+		}
+		t.Run(name, func(t *testing.T) { testReconnect(t, resumed) })
+	}
+}
+func TestRemoteSessionExpiry(t *testing.T) {
+	testReconnectExpiry(t, false, time.Second)
+}
+func testReconnect(t *testing.T, expectResumed bool) { testReconnectExpiry(t, expectResumed, 0) }
+func testReconnectExpiry(t *testing.T, expectResumed bool, expiry time.Duration) {
+	cfg := remoteConfig(t, "test-backend", false)
+	if !expectResumed {
+		cfg.MQTT.SessionExpiry = expiry
+	}
+	target := strings.TrimPrefix(cfg.MQTT.URL, "tls://")
+	proxy := newRelay(t, target)
+	defer proxy.close()
+	// The generated server certificate explicitly includes 127.0.0.1 for health checks.
+	cfg.MQTT.URL = "tls://" + proxy.listener.Addr().String()
+	cfg.MQTT.ReconnectMin = 100 * time.Millisecond
+	cfg.MQTT.ReconnectMax = 300 * time.Millisecond
+	client := mqtt.New(cfg, store(cfg))
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if err := client.Start(ctx, []string{"acks", "telemetry"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+	b := backend.New(client)
+	workerCtx, stop := context.WithCancel(ctx)
+	workerDone := make(chan error, 1)
+	go func() { workerDone <- b.Serve(workerCtx) }()
+	defer func() { stop(); <-workerDone }()
+	simCfg := remoteConfig(t, "sim-001", false)
+	simCfg.Simulator.AckDelay = 500 * time.Millisecond
+	simClient := mqtt.New(simCfg, store(simCfg))
+	sim := simulator.New(simClient)
+	simCtx, simStop := context.WithCancel(ctx)
+	simDone := make(chan error, 1)
+	go func() { simDone <- sim.Run(simCtx) }()
+	defer func() { simStop(); <-simDone }()
+	wait(t, simClient.Ready)
+	cmdDone := make(chan error, 1)
+	go func() {
+		_, err := b.Command(ctx, backend.NewCommand("sim-001", "ping", backend.ID(), 5*time.Second))
+		cmdDone <- err
+	}()
+	wait(t, func() bool { _, counts := client.Events.Snapshot(); return counts[events.CommandAccepted] > 0 })
+	proxy.pause()
+	wait(t, func() bool { return !client.Ready() })
+	select {
+	case err := <-cmdDone:
+		if err == nil || !strings.Contains(err.Error(), "unknown") {
+			t.Fatal("connection loss did not produce unknown outcome")
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("pending request did not stop on connection loss")
+	}
+	if expiry > 0 {
+		time.Sleep(expiry + 500*time.Millisecond)
+	}
+	proxy.resume()
+	wait(t, client.Ready)
+	history, _ := client.Events.Snapshot()
+	resumed := false
+	for _, e := range history {
+		if e.Kind == events.ConnectionUp && strings.Contains(e.Outcome, "session_present=true") {
+			resumed = true
+		}
+	}
+	if resumed != expectResumed {
+		t.Fatal("unexpected broker session present result")
+	}
+	ack, err := b.Command(ctx, backend.NewCommand("sim-001", "get_version", backend.ID(), 5*time.Second))
+	if err != nil || ack == nil || ack.Result.Version == "" {
+		t.Fatal("command did not recover after reconnect")
+	}
+	t.Log("actual connection loss -> unknown, TLS reconnect, session-dependent subscription restoration and command recovery: PASS")
+}

internal/integration/reliability_test.go

@@ -0,0 +1,117 @@
+package integration
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/eclipse/paho.golang/paho"
+	"robot-market/test-mqtt-5/internal/backend"
+	"robot-market/test-mqtt-5/internal/contracts"
+	"robot-market/test-mqtt-5/internal/events"
+	"robot-market/test-mqtt-5/internal/mqtt"
+)
+
+func TestRemoteBrokerQueuedMessageExpiry(t *testing.T) {
+	deviceCfg := remoteConfig(t, "sim-001", false)
+	deviceCfg.MQTT.ClientID = "rm5-expiry-" + backend.ID()[:12]
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	device := mqtt.New(deviceCfg, store(deviceCfg))
+	if err := device.Start(ctx, []string{"commands"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := device.Close(); err != nil {
+		t.Fatal(err)
+	}
+	serverCfg := remoteConfig(t, "test-backend", false)
+	server := mqtt.New(serverCfg, store(serverCfg))
+	if err := server.Start(ctx, []string{"acks"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	defer server.Close()
+	// Application deadline remains valid; the broker must expire this queued packet.
+	p := mqtt.Packet(mqtt.Topic(serverCfg, "commands"), 1, false, backend.NewCommand("sim-001", "ping", backend.ID(), 15*time.Second))
+	p.Properties.MessageExpiry = paho.Uint32(1)
+	if err := server.Publish(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * time.Second)
+	deviceCfg.MQTT.CleanStart = false
+	device = mqtt.New(deviceCfg, store(deviceCfg))
+	if err := device.Start(ctx, []string{"commands"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	defer device.Close()
+	history, _ := device.Events.Snapshot()
+	resumed := false
+	for _, e := range history {
+		if e.Kind == events.ConnectionUp && strings.Contains(e.Outcome, "session_present=true") {
+			resumed = true
+		}
+	}
+	if !resumed {
+		t.Fatal("offline subscriber session was lost; expiry test inconclusive")
+	}
+	select {
+	case <-device.Incoming:
+		t.Fatal("broker delivered expired queued command")
+	case <-time.After(300 * time.Millisecond):
+	}
+	fresh := mqtt.Packet(mqtt.Topic(serverCfg, "commands"), 1, false, backend.NewCommand("sim-001", "ping", backend.ID(), 15*time.Second))
+	if err := server.Publish(ctx, fresh); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case got := <-device.Incoming:
+		if string(got.Payload) != string(fresh.Payload) {
+			t.Fatal("wrong queued packet")
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("fresh control command missing")
+	}
+	t.Log("broker expiry distinct from valid application deadline, persistent offline queue, fresh control delivery: PASS")
+}
+
+func TestRemoteWillAfterAbruptDisconnect(t *testing.T) {
+	observerCfg := remoteConfig(t, "test-backend", false)
+	observer := mqtt.New(observerCfg, store(observerCfg))
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if err := observer.Start(ctx, []string{"status"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	defer observer.Close()
+	cfg := remoteConfig(t, "sim-001", false)
+	proxy := newRelay(t, strings.TrimPrefix(cfg.MQTT.URL, "tls://"))
+	defer proxy.close()
+	cfg.MQTT.URL = "tls://" + proxy.listener.Addr().String()
+	cfg.MQTT.ReconnectMin = 100 * time.Millisecond
+	cfg.MQTT.ReconnectMax = 200 * time.Millisecond
+	boot := "will-" + backend.ID()[:12]
+	status := contracts.Status{Envelope: contracts.Envelope{SchemaVersion: 1, DeviceID: "sim-001"}, BootID: boot, State: "offline", WillConfiguredAt: backend.Timestamp(time.Now())}
+	packet := mqtt.Packet(mqtt.Topic(cfg, "status"), 1, false, status)
+	client := mqtt.New(cfg, store(cfg))
+	if err := client.Start(ctx, []string{"commands"}, &paho.WillMessage{Topic: packet.Topic, Payload: packet.Payload, QoS: 1, Retain: false}); err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+	proxy.pause()
+	deadline := time.NewTimer(5 * time.Second)
+	defer deadline.Stop()
+	for {
+		select {
+		case p := <-observer.Incoming:
+			if strings.Contains(string(p.Payload), boot) {
+				if !strings.Contains(string(p.Payload), "will_configured_at") {
+					t.Fatal("Will invents disconnect timestamp")
+				}
+				t.Log("broker emits non-retained Will after actual abrupt TLS connection loss: PASS")
+				return
+			}
+		case <-deadline.C:
+			t.Fatal("Will not observed after abrupt connection loss")
+		}
+	}
+}