Refactor MQTT course and bound backend message processing

Move lesson source into the teaching structure, align the course path with the learner's Go and DevOps experience, and add the interactive lesson guides. Add advanced DevOps lifecycle study as a separate section. Bound MQTT message work and drain accepted jobs during shutdown; align the backend implementation with the revised course through Section 6.
2026-10-22 10:19:56 +03:30
parent f9ab91ed60
commit fb30e8e884
70 changed files with 7492 additions and 312 deletions
@@ -5,12 +5,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"log/slog"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"

 	"example.com/mqtt5-learning/internal/config"
+	"example.com/mqtt5-learning/internal/contracts"
+	"example.com/mqtt5-learning/internal/deviceview"
+	"example.com/mqtt5-learning/internal/dispatch"
 	"example.com/mqtt5-learning/internal/mqtt"
 )

@@ -26,47 +30,86 @@ func run() error {
 		return fmt.Errorf("failed to load config: %w", err)
 	}

-	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	lifecycleCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
+	transportCtx, stopTransport := context.WithCancel(context.Background())
+	defer stopTransport()

-	base := "lab/v1/devices/" + cfg.MQTT.DeviceID
+	store := deviceview.NewStore()
+	service, err := deviceview.NewService(store, slog.Default(), 90*time.Second)
+	if err != nil {
+		return fmt.Errorf("create device-view service: %w", err)
+	}
+
+	telemetryFilter, err := contracts.DeviceFilter(contracts.KindTelemetry)
+	if err != nil {
+		return err
+	}
+	responsesFilter, err := contracts.DeviceFilter(contracts.KindResponses)
+	if err != nil {
+		return err
+	}
+	commandTopic, err := contracts.DeviceTopic(cfg.MQTT.DeviceID, contracts.KindCommands)
+	if err != nil {
+		return err
+	}
 	inbox := make(chan mqtt.Message, 16)
-
-	manager, err := mqtt.Start(ctx, cfg, []string{base + "/telemetry", base + "/responses"}, inbox)
+	manager, err := mqtt.Start(transportCtx, cfg, []contracts.TopicFilter{telemetryFilter, responsesFilter}, inbox)
 	if err != nil {
 		return fmt.Errorf("failed to start MQTT client: %w", err)
 	}
 	defer mqtt.Stop(manager)

+	telemetryPool := dispatch.New(context.Background(), 2, 16)
+	importantPool := dispatch.New(context.Background(), 2, 32)
+	receiver := &dispatch.Receiver{
+		Telemetry: telemetryPool,
+		Important: importantPool,
+		Process:   service.HandleMessage,
+	}
+	defer func() {
+		drainCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		started := time.Now()
+		if err := dispatch.Shutdown(drainCtx, telemetryPool, importantPool); err != nil {
+			log.Printf("dispatcher shutdown incomplete: %v", err)
+		}
+		log.Printf("dispatcher drain finished elapsed=%s telemetry=%+v important=%+v",
+			time.Since(started), telemetryPool.Stats(), importantPool.Stats())
+	}()
+
 	ticker := time.NewTicker(10 * time.Second)
 	defer ticker.Stop()
-
 	runID := time.Now().UnixNano()
 	sequence := 0

 	for {
 		select {
-		case <-ctx.Done():
-			log.Print("shutting down")
+		case <-lifecycleCtx.Done():
+			log.Print("shutting down; draining accepted work before MQTT disconnect")
 			return nil
-		case msg := <-inbox:
-			log.Printf("received message on topic %s retained=%t payload=%s", msg.Topic, msg.Retain, string(msg.Payload))
-
+		case message := <-inbox:
+			outcome, submitErr := receiver.Submit(lifecycleCtx, message)
+			if submitErr != nil || outcome != dispatch.Accepted {
+				slog.Warn("message admission rejected", "topic", message.Topic, "outcome", outcome, "error", submitErr)
+			}
 		case <-ticker.C:
 			sequence++
-			payload, err := json.Marshal(struct {
-				ID      string `json:"id"`
-				Command string `json:"command"`
-			}{
-				ID: fmt.Sprintf("ping-%d-%d", runID, sequence), Command: "ping",
+			id := fmt.Sprintf("ping-%d-%d", runID, sequence)
+			payload, marshalErr := json.Marshal(contracts.Command{
+				SchemaVersion: contracts.SchemaVersionOne,
+				MessageID:     id,
+				DeviceID:      cfg.MQTT.DeviceID,
+				OccurredAt:    time.Now().UTC().Format(time.RFC3339),
+				CommandID:     id,
+				Command:       "ping",
 			})
-			if err != nil {
-				return fmt.Errorf("failed to marshal payload: %w", err)
+			if marshalErr != nil {
+				return fmt.Errorf("failed to marshal payload: %w", marshalErr)
 			}
-			if err := mqtt.Publish(ctx, manager, base+"/commands", payload); err != nil {
-				log.Printf("command publication failed or outcome uncertain: %v", err)
+			if publishErr := mqtt.Publish(lifecycleCtx, manager, commandTopic, payload); publishErr != nil {
+				log.Printf("command publication failed or outcome uncertain: %v", publishErr)
 			}
 		}
 	}
-
 }
@@ -3,14 +3,19 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"log"
 	"os"
 	"os/signal"
+	"sync/atomic"
 	"syscall"
 	"time"

 	"example.com/mqtt5-learning/internal/config"
+	"example.com/mqtt5-learning/internal/contracts"
+	"example.com/mqtt5-learning/internal/dispatch"
 	mqtt "example.com/mqtt5-learning/internal/mqtt"
+	"github.com/eclipse/paho.golang/autopaho"
 )

 func main() {
@@ -29,59 +34,111 @@ func run() error {
 	}
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
-	// Build this device's topic namespace and bounded receive queue.
-	base := "lab/v1/devices/" + cfg.MQTT.DeviceID
+	transportCtx, stopTransport := context.WithCancel(context.Background())
+	defer stopTransport()
+	// Build concrete names for this device and a filter for its command flow.
+	commandFilter, err := contracts.DeviceSubscription(cfg.MQTT.DeviceID, contracts.KindCommands)
+	if err != nil {
+		return err
+	}
+	telemetryTopic, err := contracts.DeviceTopic(cfg.MQTT.DeviceID, contracts.KindTelemetry)
+	if err != nil {
+		return err
+	}
 	inbox := make(chan mqtt.Message, 16)
-	manager, err := mqtt.Start(ctx, cfg, []string{base + "/commands"}, inbox)
+	manager, err := mqtt.Start(transportCtx, cfg, []contracts.TopicFilter{commandFilter}, inbox)
 	if err != nil {
 		return err
 	}
 	defer mqtt.Stop(manager)
+
+	// Keep worker contexts alive during the drain; the signal context stops intake.
+	runID := time.Now().UnixNano()
+	telemetryPool := dispatch.New(context.Background(), 1, 1)
+	importantPool := dispatch.New(context.Background(), 2, 16)
+	var resultSequence atomic.Uint64
+	receiver := &dispatch.Receiver{
+		Telemetry: telemetryPool,
+		Important: importantPool,
+		Process: func(jobCtx context.Context, message mqtt.Message) error {
+			resultID := fmt.Sprintf("result-%d-%d", runID, resultSequence.Add(1))
+			return respond(jobCtx, manager, cfg.MQTT.DeviceID, resultID, message)
+		},
+	}
+	defer func() {
+		drainCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		started := time.Now()
+		if err := dispatch.Shutdown(drainCtx, telemetryPool, importantPool); err != nil {
+			log.Printf("dispatcher shutdown incomplete: %v", err)
+		}
+		log.Printf("dispatcher drain finished elapsed=%s telemetry=%+v important=%+v",
+			time.Since(started), telemetryPool.Stats(), importantPool.Stats())
+	}()
 	// Heartbeats are periodic; command handling remains active between ticks.
 	ticker := time.NewTicker(3 * time.Second)
 	defer ticker.Stop()
+	var heartbeatSequence uint64
 	for {
 		// Handle cancellation, telemetry publication, and inbound commands together.
 		select {
 		case <-ctx.Done():
+			log.Print("shutting down; draining accepted command work before MQTT disconnect")
 			return nil
 		case <-ticker.C:
-			payload, err := json.Marshal(struct {
-				DeviceID string `json:"device_id"`
-				Time     string `json:"time"`
-			}{cfg.MQTT.DeviceID, time.Now().UTC().Format(time.RFC3339)})
+			heartbeatSequence++
+			now := time.Now().UTC()
+			payload, err := json.Marshal(contracts.Heartbeat{
+				SchemaVersion: contracts.SchemaVersionOne,
+				MessageID:     fmt.Sprintf("heartbeat-%d-%d", runID, heartbeatSequence),
+				DeviceID:      cfg.MQTT.DeviceID,
+				OccurredAt:    now.Format(time.RFC3339),
+			})
 			if err != nil {
 				return err
 			}
-			if err := mqtt.Publish(ctx, manager, base+"/telemetry", payload); err != nil {
+			if err = mqtt.Publish(ctx, manager, telemetryTopic, payload); err != nil {
 				log.Printf("heartbeat: %v", err)
 			}
 		case message := <-inbox:
-			// Decode into a local command value only after rejecting retained messages.
-			var command struct {
-				ID      string `json:"id"`
-				Command string `json:"command"`
-			}
-			if message.Retain {
-				log.Print("ignoring retained command")
-				continue
-			}
-			if err := json.Unmarshal(message.Payload, &command); err != nil || command.ID == "" || command.Command != "ping" {
-				log.Print("ignoring malformed or unsupported command")
-				continue
-			}
-			// Echo the command ID so the backend can associate this pong with its ping.
-			response, err := json.Marshal(struct {
-				ID       string `json:"id"`
-				DeviceID string `json:"device_id"`
-				Status   string `json:"status"`
-			}{command.ID, cfg.MQTT.DeviceID, "pong"})
-			if err != nil {
-				return err
-			}
-			if err := mqtt.Publish(ctx, manager, base+"/responses", response); err != nil {
-				log.Printf("response: %v", err)
+			outcome, submitErr := receiver.Submit(ctx, message)
+			if submitErr != nil || outcome != dispatch.Accepted {
+				log.Printf("command admission rejected outcome=%d topic=%q error=%v", outcome, message.Topic, submitErr)
 			}
 		}
 	}
 }
+
+func respond(ctx context.Context, manager *autopaho.ConnectionManager, deviceID, resultMessageID string, message mqtt.Message) error {
+	if message.Retain {
+		return fmt.Errorf("retained command rejected")
+	}
+	topic, err := contracts.ParseTopic(message.Topic)
+	if err != nil {
+		return err
+	}
+	if topic.Kind != contracts.KindCommands || topic.DeviceID != deviceID {
+		return fmt.Errorf("command topic is not authorized for this device")
+	}
+	command, err := contracts.Decode[contracts.Command](message.Payload, topic, time.Now().UTC())
+	if err != nil {
+		return err
+	}
+	response, err := json.Marshal(contracts.CommandResult{
+		SchemaVersion: contracts.SchemaVersionOne,
+		MessageID:     resultMessageID,
+		DeviceID:      deviceID,
+		OccurredAt:    time.Now().UTC().Format(time.RFC3339),
+		CommandID:     command.CommandID,
+		Status:        "completed",
+		Detail:        "pong",
+	})
+	if err != nil {
+		return fmt.Errorf("encode command result: %w", err)
+	}
+	responseTopic, err := contracts.DeviceTopic(deviceID, contracts.KindResponses)
+	if err != nil {
+		return fmt.Errorf("build response topic: %w", err)
+	}
+	return mqtt.Publish(ctx, manager, responseTopic, response)
+}