Deploy MQTT test backend with private viewer and Gitea CI/CD

2026-10-08 14:48:53 +03:30
commit 71ef622b4f
70 changed files with 7123 additions and 0 deletions
@@ -0,0 +1,417 @@
+// Package config loads and validates configuration without opening network connections.
+package config
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/knadh/koanf/parsers/toml/v2"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
+	"github.com/knadh/koanf/v2"
+)
+
+const Prefix = "TEST_MQTT5_"
+
+type Config struct {
+	MQTT      MQTT      `koanf:"mqtt"`
+	TLS       TLS       `koanf:"tls"`
+	Topics    Topics    `koanf:"topics"`
+	Deadlines Deadlines `koanf:"deadlines"`
+	Logging   Logging   `koanf:"logging"`
+	Limits    Limits    `koanf:"limits"`
+	Simulator Simulator `koanf:"simulator"`
+	Web       Web       `koanf:"web"`
+	Receipts  Receipts  `koanf:"receipts"`
+}
+type MQTT struct {
+	AllowPlaintext bool          `koanf:"allow_plaintext"`
+	URL            string        `koanf:"url"`
+	ClientID       string        `koanf:"client_id"`
+	Username       string        `koanf:"username"`
+	Password       string        `koanf:"password"`
+	PasswordFile   string        `koanf:"password_file"`
+	QoS            int           `koanf:"qos"`
+	CleanStart     bool          `koanf:"clean_start"`
+	SessionExpiry  time.Duration `koanf:"session_expiry"`
+	KeepAlive      time.Duration `koanf:"keep_alive"`
+	ReconnectMin   time.Duration `koanf:"reconnect_min"`
+	ReconnectMax   time.Duration `koanf:"reconnect_max"`
+}
+type TLS struct {
+	CAFile     string `koanf:"ca_file"`
+	ClientCert string `koanf:"client_cert"`
+	ClientKey  string `koanf:"client_key"`
+}
+type Topics struct {
+	Namespace      string `koanf:"namespace"`
+	DeviceID       string `koanf:"device_id"`
+	ObserveDevices string `koanf:"observe_devices"`
+	ObserveAll     bool   `koanf:"observe_all"`
+}
+type Deadlines struct {
+	Connect   time.Duration `koanf:"connect"`
+	Operation time.Duration `koanf:"operation"`
+	Command   time.Duration `koanf:"command"`
+	Shutdown  time.Duration `koanf:"shutdown"`
+}
+type Logging struct {
+	Level       string `koanf:"level"`
+	Format      string `koanf:"format"`
+	RawPayloads bool   `koanf:"raw_payloads"`
+}
+type Limits struct {
+	PayloadBytes    int           `koanf:"payload_bytes"`
+	PreviewBytes    int           `koanf:"preview_bytes"`
+	PendingCommands int           `koanf:"pending_commands"`
+	InboundQueue    int           `koanf:"inbound_queue"`
+	EventHistory    int           `koanf:"event_history"`
+	DedupEntries    int           `koanf:"dedup_entries"`
+	DedupRetention  time.Duration `koanf:"dedup_retention"`
+}
+type Simulator struct {
+	TelemetryInterval time.Duration `koanf:"telemetry_interval"`
+	AckDelay          time.Duration `koanf:"ack_delay"`
+}
+type Receipts struct {
+	Enabled bool `koanf:"enabled"`
+}
+type Web struct {
+	Enabled     bool   `koanf:"enabled"`
+	Address     string `koanf:"address"`
+	MaxViewers  int    `koanf:"max_viewers"`
+	ViewerQueue int    `koanf:"viewer_queue"`
+}
+
+func Defaults() Config {
+	return Config{
+		MQTT:      MQTT{URL: "tls://localhost:8883", ClientID: "rm5-lab-observer-01", Username: "test-backend", QoS: 1, CleanStart: true, SessionExpiry: 5 * time.Minute, KeepAlive: 30 * time.Second, ReconnectMin: time.Second, ReconnectMax: 30 * time.Second},
+		Topics:    Topics{Namespace: "robot-market/test/v1", DeviceID: "sim-001"},
+		Deadlines: Deadlines{Connect: 10 * time.Second, Operation: 5 * time.Second, Command: 15 * time.Second, Shutdown: 5 * time.Second},
+		Logging:   Logging{Level: "info", Format: "text"},
+		Limits:    Limits{PayloadBytes: 16384, PreviewBytes: 4096, PendingCommands: 128, InboundQueue: 256, EventHistory: 1000, DedupEntries: 1024, DedupRetention: 10 * time.Minute},
+		Simulator: Simulator{TelemetryInterval: 5 * time.Second},
+		Web:       Web{Address: "127.0.0.1:8080", MaxViewers: 8, ViewerQueue: 64},
+	}
+}
+
+// Load uses only explicitly supplied overrides. Relative paths are relative to cwd.
+// Errors deliberately exclude raw parser errors and values, which may contain secrets.
+func Load(path string, overrides map[string]string) (Config, error) {
+	return LoadWithDefaults(path, overrides, Defaults())
+}
+
+// LoadWithDefaults applies role defaults below TOML, environment and explicit flags.
+func LoadWithDefaults(path string, overrides map[string]string, cfg Config) (Config, error) {
+	fields := map[string]reflect.Value{}
+	collect(reflect.ValueOf(&cfg).Elem(), "", fields)
+	k := koanf.New(".")
+	for key, v := range fields {
+		value := v.Interface()
+		if v.Type() == reflect.TypeFor[time.Duration]() {
+			value = time.Duration(v.Int()).String()
+		}
+		if err := k.Set(key, value); err != nil {
+			return Config{}, fmt.Errorf("defaults: unable to set configuration")
+		}
+	}
+	if path != "" {
+		info, err := os.Stat(path)
+		if err != nil || !info.Mode().IsRegular() {
+			return Config{}, fmt.Errorf("config: file must exist and be a readable regular file")
+		}
+		if info.Size() > 1<<20 {
+			return Config{}, fmt.Errorf("config: file exceeds 1 MiB")
+		}
+		data, err := file.Provider(path).ReadBytes()
+		if err != nil {
+			return Config{}, fmt.Errorf("config: unable to read file")
+		}
+		if len(data) > 1<<20 {
+			return Config{}, fmt.Errorf("config: file exceeds 1 MiB")
+		}
+		values, err := toml.Parser().Unmarshal(data)
+		if err != nil {
+			return Config{}, fmt.Errorf("config: invalid TOML; check syntax (values omitted)")
+		}
+		if err = checkKeys(values, "", fields); err != nil {
+			return Config{}, err
+		}
+		if err = setValues(k, values, ""); err != nil {
+			return Config{}, fmt.Errorf("config: unable to merge TOML")
+		}
+	}
+	// Read flat keys to reject ambiguous names before unflattening.
+	values, err := env.Provider(Prefix, "", func(key string) string {
+		return strings.ReplaceAll(strings.ToLower(strings.TrimPrefix(key, Prefix)), "__", ".")
+	}).Read()
+	if err != nil {
+		return Config{}, fmt.Errorf("environment: unable to read settings")
+	}
+	for key, value := range values {
+		if _, ok := fields[key]; !ok {
+			return Config{}, unknownKey(key)
+		}
+		if err = k.Set(key, value); err != nil {
+			return Config{}, fmt.Errorf("environment: unable to merge settings")
+		}
+	}
+	for key, value := range overrides {
+		if _, ok := fields[key]; !ok {
+			return Config{}, unknownKey(key)
+		}
+		if err = k.Set(key, value); err != nil {
+			return Config{}, fmt.Errorf("flags: unable to merge settings")
+		}
+	}
+	for key, field := range fields {
+		if err := assign(field, k.Get(key)); err != nil {
+			return Config{}, fmt.Errorf("%s: %s", key, err)
+		}
+	}
+	if err := cfg.Validate(); err != nil {
+		return Config{}, err
+	}
+	return cfg, nil
+}
+
+func setValues(k *koanf.Koanf, values map[string]any, prefix string) error {
+	for key, value := range values {
+		if nested, ok := value.(map[string]any); ok {
+			if err := setValues(k, nested, prefix+key+"."); err != nil {
+				return err
+			}
+		} else if err := k.Set(prefix+key, value); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func collect(v reflect.Value, prefix string, out map[string]reflect.Value) {
+	for i := 0; i < v.NumField(); i++ {
+		key := prefix + v.Type().Field(i).Tag.Get("koanf")
+		f := v.Field(i)
+		if f.Kind() == reflect.Struct {
+			collect(f, key+".", out)
+		} else {
+			out[key] = f
+		}
+	}
+}
+func unknownKey(key string) error {
+	if !regexp.MustCompile(`^[a-z_][a-z0-9_.]{0,80}$`).MatchString(key) {
+		return fmt.Errorf("configuration: invalid or unknown key (name omitted)")
+	}
+	return fmt.Errorf("%s: unknown configuration key", key)
+}
+func checkKeys(values map[string]any, prefix string, fields map[string]reflect.Value) error {
+	for key, value := range values {
+		full := prefix + key
+		if nested, ok := value.(map[string]any); ok {
+			known := false
+			for field := range fields {
+				if strings.HasPrefix(field, full+".") {
+					known = true
+					break
+				}
+			}
+			if !known {
+				return unknownKey(full)
+			}
+			if err := checkKeys(nested, full+".", fields); err != nil {
+				return err
+			}
+		} else if _, ok := fields[full]; !ok {
+			return unknownKey(full)
+		}
+	}
+	return nil
+}
+func assign(field reflect.Value, value any) error {
+	if field.Type() == reflect.TypeFor[time.Duration]() {
+		s, ok := value.(string)
+		if !ok {
+			return fmt.Errorf("expected duration string with unit, e.g. 5s")
+		}
+		if !strings.ContainsAny(s, "nhµμmsu") {
+			return fmt.Errorf("expected duration string with unit, e.g. 5s")
+		}
+		d, err := time.ParseDuration(s)
+		if err != nil {
+			return fmt.Errorf("invalid or overflowing duration")
+		}
+		field.SetInt(int64(d))
+		return nil
+	}
+	switch field.Kind() {
+	case reflect.String:
+		s, ok := value.(string)
+		if !ok {
+			return fmt.Errorf("expected string")
+		}
+		field.SetString(s)
+	case reflect.Bool:
+		b, ok := value.(bool)
+		if !ok {
+			s, isString := value.(string)
+			if !isString || (s != "true" && s != "false") {
+				return fmt.Errorf("expected true or false")
+			}
+			b = s == "true"
+		}
+		field.SetBool(b)
+	case reflect.Int:
+		var n int64
+		switch v := value.(type) {
+		case int:
+			n = int64(v)
+		case int64:
+			n = v
+		case string:
+			parsed, err := strconv.ParseInt(v, 10, 64)
+			if err != nil {
+				return fmt.Errorf("expected integer within range")
+			}
+			n = parsed
+		default:
+			return fmt.Errorf("expected integer")
+		}
+		if field.OverflowInt(n) {
+			return fmt.Errorf("integer out of range")
+		}
+		field.SetInt(n)
+	default:
+		return fmt.Errorf("unsupported configuration type")
+	}
+	return nil
+}
+
+var identifier = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$`)
+
+func (c Config) Validate() error {
+	bad := func(key, why string) error { return fmt.Errorf("%s: %s", key, why) }
+	u, err := url.Parse(c.MQTT.URL)
+	if err != nil || (u.Scheme != "tls" && !(u.Scheme == "mqtt" && c.MQTT.AllowPlaintext)) || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
+		return bad("mqtt.url", "use tls://host:port; mqtt:// requires mqtt.allow_plaintext=true; omit credentials, path, query and fragment")
+	}
+	port, err := strconv.Atoi(u.Port())
+	if err != nil || port < 1 || port > 65535 {
+		return bad("mqtt.url", "explicit port must be 1..65535")
+	}
+	if !identifier.MatchString(c.MQTT.ClientID) {
+		return bad("mqtt.client_id", "use 1..64 letters, digits, underscores or hyphens, starting with a letter/digit")
+	}
+	if !identifier.MatchString(c.Topics.DeviceID) {
+		return bad("topics.device_id", "use 1..64 safe identifier characters")
+	}
+	if c.Topics.ObserveDevices != "" {
+		devices := strings.Split(c.Topics.ObserveDevices, ",")
+		if len(devices) > 16 {
+			return bad("topics.observe_devices", "maximum 16 device IDs")
+		}
+		for _, device := range devices {
+			if !identifier.MatchString(strings.TrimSpace(device)) {
+				return bad("topics.observe_devices", "use comma-separated safe device IDs")
+			}
+		}
+	}
+	if c.Topics.Namespace != "robot-market/test/v1" && !strings.HasPrefix(c.Topics.Namespace, "robot-market/test/v1/") {
+		return bad("topics.namespace", "must remain under robot-market/test/v1")
+	}
+	if len(c.Topics.Namespace) > 200 {
+		return bad("topics.namespace", "too long")
+	}
+	for _, part := range strings.Split(c.Topics.Namespace, "/") {
+		if !identifier.MatchString(part) {
+			return bad("topics.namespace", "segments must be nonempty safe identifiers")
+		}
+	}
+	if c.MQTT.QoS < 0 || c.MQTT.QoS > 1 {
+		return bad("mqtt.qos", "only 0 or 1 supported in this test environment")
+	}
+	if c.MQTT.Username == "" || len(c.MQTT.Username) > 128 {
+		return bad("mqtt.username", "required, maximum 128 bytes")
+	}
+	if c.MQTT.Password != "" && c.MQTT.PasswordFile != "" {
+		return bad("mqtt.password_file", "cannot be combined with mqtt.password")
+	}
+	if len(c.MQTT.Password) > 4096 {
+		return bad("mqtt.password", "maximum 4096 bytes")
+	}
+	if (c.TLS.ClientCert == "") != (c.TLS.ClientKey == "") {
+		return bad("tls.client_cert", "client_cert and client_key must be supplied together")
+	}
+	for key, path := range map[string]string{"tls.ca_file": c.TLS.CAFile, "tls.client_cert": c.TLS.ClientCert, "tls.client_key": c.TLS.ClientKey, "mqtt.password_file": c.MQTT.PasswordFile} {
+		if path != "" {
+			f, err := os.Open(path)
+			if err != nil {
+				return bad(key, "file must exist and be readable")
+			}
+			info, err := f.Stat()
+			f.Close()
+			if err != nil || !info.Mode().IsRegular() || info.Size() == 0 || info.Size() > 1<<20 {
+				return bad(key, "expected nonempty regular file up to 1 MiB")
+			}
+		}
+	}
+	if c.MQTT.SessionExpiry < 0 || c.MQTT.SessionExpiry%time.Second != 0 || c.MQTT.SessionExpiry/time.Second > 1<<32-1 {
+		return bad("mqtt.session_expiry", "must be whole seconds in 0..4294967295")
+	}
+	if c.MQTT.KeepAlive < time.Second || c.MQTT.KeepAlive%time.Second != 0 || c.MQTT.KeepAlive/time.Second > 65535 {
+		return bad("mqtt.keep_alive", "must be whole seconds in 1..65535")
+	}
+	for key, d := range map[string]time.Duration{"mqtt.reconnect_min": c.MQTT.ReconnectMin, "mqtt.reconnect_max": c.MQTT.ReconnectMax, "deadlines.connect": c.Deadlines.Connect, "deadlines.operation": c.Deadlines.Operation, "deadlines.shutdown": c.Deadlines.Shutdown, "simulator.telemetry_interval": c.Simulator.TelemetryInterval, "limits.dedup_retention": c.Limits.DedupRetention} {
+		if d <= 0 || d > 24*time.Hour {
+			return bad(key, "must be positive and no greater than 24h")
+		}
+	}
+	if c.MQTT.ReconnectMin > c.MQTT.ReconnectMax || c.MQTT.ReconnectMax > 30*time.Second {
+		return bad("mqtt.reconnect_max", "must be >= reconnect_min and <= 30s")
+	}
+	if c.Deadlines.Command < time.Second || c.Deadlines.Command > time.Minute || c.Deadlines.Command%time.Second != 0 {
+		return bad("deadlines.command", "must be whole seconds in 1s..1m")
+	}
+	if c.Limits.DedupRetention <= c.Deadlines.Command {
+		return bad("limits.dedup_retention", "must exceed command lifetime")
+	}
+	if c.Simulator.AckDelay < 0 || c.Simulator.AckDelay > time.Hour {
+		return bad("simulator.ack_delay", "must be in 0s..1h")
+	}
+	if c.Logging.Level != "debug" && c.Logging.Level != "info" && c.Logging.Level != "warn" && c.Logging.Level != "error" {
+		return bad("logging.level", "must be debug, info, warn or error")
+	}
+	if c.Logging.Format != "text" && c.Logging.Format != "json" {
+		return bad("logging.format", "must be text or json")
+	}
+	for key, n := range map[string]int{"limits.pending_commands": c.Limits.PendingCommands, "limits.inbound_queue": c.Limits.InboundQueue, "limits.event_history": c.Limits.EventHistory, "limits.dedup_entries": c.Limits.DedupEntries, "web.viewer_queue": c.Web.ViewerQueue} {
+		if n < 1 || n > 10000 {
+			return bad(key, "must be in 1..10000")
+		}
+	}
+	if c.Limits.PayloadBytes < 256 || c.Limits.PayloadBytes > 16384 {
+		return bad("limits.payload_bytes", "must be in 256..16384")
+	}
+	if c.Limits.PreviewBytes < 64 || c.Limits.PreviewBytes > 4096 || c.Limits.PreviewBytes > c.Limits.PayloadBytes {
+		return bad("limits.preview_bytes", "must be 64..4096 and no larger than payload_bytes")
+	}
+	if c.Web.ViewerQueue < 1 || c.Web.ViewerQueue > 256 {
+		return bad("web.viewer_queue", "must be in 1..256")
+	}
+	if c.Web.MaxViewers < 1 || c.Web.MaxViewers > 8 {
+		return bad("web.max_viewers", "must be 1..8")
+	}
+	host, portText, err := net.SplitHostPort(c.Web.Address)
+	p, portErr := strconv.Atoi(portText)
+	if err != nil || host != "127.0.0.1" || portErr != nil || p < 1 || p > 65535 {
+		return bad("web.address", "use 127.0.0.1 with a port in 1..65535")
+	}
+	return nil
+}