feat(cloud): docktail.cloud.* labels for monitoring intent

cloud/checks.go

@@ -50,8 +50,9 @@ type serviceCheck struct {
 }
 
 // run probes every service once. Cloud check config (keyed by service key)
-// selects the check shape but never its destination; services with no locally
-// discovered target are skipped.
+// selects the check shape but never its destination, and the service's own
+// docktail.cloud.check.* labels override it field by field; services with no
+// locally discovered target are skipped.
 func (c *checker) run(ctx context.Context, services []proto.Service, configs []proto.CheckConfig) []proto.CheckResult {
 	configs, _ = proto.SanitizeCheckConfigs(configs)
 	cfgByKey := make(map[string]proto.CheckConfig, len(configs))
@@ -66,6 +67,10 @@ func (c *checker) run(ctx context.Context, services []proto.Service, configs []p
 			cfg := cc
 			sc.cfg = &cfg
 		}
+		// docktail.cloud.check.* labels win over the cloud's config, field by field.
+		if merged, ok := applyLabelIntent(svc.Key, sc.cfg, svc.LabelIntent); ok {
+			sc.cfg = &merged
+		}
 		if res, ok := c.runOne(ctx, sc); ok {
 			results = append(results, res)
 		}

cloud/collector.go

@@ -56,6 +56,7 @@ type Collector struct {
 	logMode      string              // workspace default capture mode ("" ⇒ proto.LogModeOff)
 	logOverrides map[string]string   // per-service capture mode override (service key -> proto.LogMode*)
 	checkFails   map[string]int      // consecutive local-check failures per service key, for incident log capture
+	labelWarned  map[string]string   // container id -> the docktail.cloud.* label problems last warned about
 	cfgVer       int
 	unmonitored  bool      // cloud reports this host inactive/past the plan cap; throttle output
 	lastTeaser   time.Time // last throttled teaser snapshot sent while unmonitored
@@ -132,6 +133,7 @@ func NewCollector(ctx context.Context, cfg Config, dc *docker.Client, ts tailnet
 		specs:          specs,
 		logOverrides:   map[string]string{},
 		checkFails:     map[string]int{},
+		labelWarned:    map[string]string{},
 		oomSeen:        map[string]time.Time{},
 		prevCPU:        map[string]cpuSample{},
 		prevCPUOther:   map[string]cpuSample{},
@@ -162,7 +164,7 @@ func (c *Collector) Fingerprint() string { return c.fingerprint }
 // OnReconcile receives the reconciler's freshly computed services, enriches them
 // with runtime detail, stores them, and (if connected) sends a snapshot.
 func (c *Collector) OnReconcile(ctx context.Context, services []*apptypes.ContainerService) {
-	built := c.buildServices(ctx, services)
+	built := c.buildServices(ctx, services, false)
 
 	c.mu.Lock()
 	c.latest = built
@@ -203,8 +205,14 @@ func (c *Collector) OnEvent(ctx context.Context, msg events.Message) {
 	if unmonitored {
 		return
 	}
+	// The event's attributes carry the container's labels, so a label opt-out
+	// holds even before the container's first snapshot.
+	noCapture := labelsForbidCapture(msg.Actor.Attributes)
 	for _, ev := range evs {
 		c.sendOrSpool(conn, proto.TypeEvent, ev)
+		if noCapture {
+			continue
+		}
 		// Capture the tail now, not at replay time: by the time the link is back the
 		// container may have been recreated and its logs gone with it.
 		c.maybeCaptureLogs(ctx, conn, ev)
@@ -215,7 +223,11 @@ func (c *Collector) OnEvent(ctx context.Context, msg events.Message) {
 
 // buildServices maps reconciler ContainerService values to wire Services,
 // enriching each with a single inspect + stats sample per distinct container.
-func (c *Collector) buildServices(ctx context.Context, services []*apptypes.ContainerService) []proto.Service {
+//
+// full marks a build from self-discovery, which lists every managed container
+// (stopped ones included); only such a build may forget label warnings for
+// containers it did not see.
+func (c *Collector) buildServices(ctx context.Context, services []*apptypes.ContainerService, full bool) []proto.Service {
 	type enriched struct {
 		info  docker.CloudInfo
 		stats containerStats
@@ -225,10 +237,21 @@ func (c *Collector) buildServices(ctx context.Context, services []*apptypes.Cont
 	// of them, and toService stays a pure mapping.
 	funnelHost := c.funnelHostname()
 	out := make([]proto.Service, 0, len(services))
+	labelled := make(map[string]struct{})
 	for _, cs := range services {
 		if cs == nil {
 			continue
 		}
+		labels := parseCloudLabels(cs.CloudLabels)
+		if _, seen := labelled[cs.ContainerID]; !seen {
+			labelled[cs.ContainerID] = struct{}{}
+			c.warnLabelProblems(cs.ContainerID, cs.ContainerName, labels.problems)
+		}
+		if labels.ignored {
+			// docktail.cloud.ignore=true: not a cloud service at all. The container
+			// is reported as plain inventory (GetOtherContainers) instead.
+			continue
+		}
 		e, ok := cache[cs.ContainerID]
 		if !ok {
 			if ci, err := c.docker.InspectCloud(ctx, cs.ContainerID); err == nil {
@@ -237,16 +260,54 @@ func (c *Collector) buildServices(ctx context.Context, services []*apptypes.Cont
 			e.stats = c.sampleStats(ctx, cs.ContainerID, e.info.State)
 			cache[cs.ContainerID] = e
 		}
-		out = append(out, toService(cs, e.info, e.stats, funnelHost))
+		svc := toService(cs, e.info, e.stats, funnelHost)
+		svc.LabelIntent = intentForService(labels.intent, cs.Protocol)
+		out = append(out, svc)
 	}
 	present := make(map[string]struct{}, len(cache))
 	for id := range cache {
 		present[id] = struct{}{}
 	}
 	c.pruneStats(present)
+	if full {
+		c.pruneLabelWarnings(labelled)
+	}
 	return out
 }
 
+// warnLabelProblems logs a container's invalid or unknown docktail.cloud.*
+// labels once, and again only when the set of problems changes, so a bad label
+// does not repeat on every discovery tick.
+func (c *Collector) warnLabelProblems(containerID, containerName string, problems []string) {
+	signature := strings.Join(problems, "\n")
+	c.mu.Lock()
+	previous, known := c.labelWarned[containerID]
+	if signature == "" {
+		delete(c.labelWarned, containerID)
+	} else {
+		c.labelWarned[containerID] = signature
+	}
+	c.mu.Unlock()
+	if signature == "" || (known && previous == signature) {
+		return
+	}
+	for _, problem := range problems {
+		c.log.Warn().Str("container", containerName).Msg("cloud: " + problem)
+	}
+}
+
+// pruneLabelWarnings forgets label warnings for containers no longer in the
+// latest build, so a recreated container warns again and the map stays bounded.
+func (c *Collector) pruneLabelWarnings(present map[string]struct{}) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for id := range c.labelWarned {
+		if _, ok := present[id]; !ok {
+			delete(c.labelWarned, id)
+		}
+	}
+}
+
 // sampleStats reads a one-shot docker stats sample for a running container and
 // turns it into a current-value usage reading. Memory is taken as-is; CPU% is
 // the delta of cumulative counters against this container's previous sample —
@@ -476,6 +537,16 @@ func (c *Collector) eventBases(msg events.Message, attrs map[string]string) []pr
 }
 
 func (c *Collector) serviceKeysForEvent(msg events.Message, attrs map[string]string) []string {
+	// An ignored container is plain inventory to the cloud, so its events name
+	// the container, never a service — not even one the same-named container
+	// published before the label was added (compose recreates under the name).
+	if docker.IsCloudIgnored(attrs) {
+		if name := strings.TrimSpace(attrs["name"]); name != "" {
+			return []string{name}
+		}
+		return nil
+	}
+
 	c.mu.RLock()
 	latest := c.latest
 	c.mu.RUnlock()
@@ -1025,7 +1096,7 @@ func (c *Collector) scanAndSnapshot(ctx context.Context, conn *wsConn) {
 		c.log.Warn().Err(err).Msg("cloud: container discovery failed")
 		return
 	}
-	built := c.buildServices(ctx, containers)
+	built := c.buildServices(ctx, containers, true)
 	c.mu.Lock()
 	c.latest = built
 	c.mu.Unlock()
@@ -1072,6 +1143,7 @@ func (c *Collector) buildContainers(ctx context.Context, containers []docker.Oth
 		out = append(out, proto.Container{
 			ContainerID:    oc.ID,
 			IsAgent:        oc.IsAgent,
+			LabelIgnored:   oc.LabelIgnored,
 			Name:           oc.Name,
 			Image:          oc.Image,
 			ImageTag:       oc.ImageTag,
@@ -1309,11 +1381,19 @@ func (c *Collector) applyConfig(cfg proto.Config) {
 		Msg("cloud: applied config")
 }
 
-// logModeFor returns the effective capture mode for a service key: its override
-// when set, else the workspace default, else the built-in default (off).
+// logModeFor returns the effective capture mode for a service key: off when its
+// container is labelled docktail.cloud.logs=off, else its cloud override when
+// set, else the workspace default, else the built-in default (off).
 func (c *Collector) logModeFor(serviceKey string) string {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
+	// docktail.cloud.logs=off on the service's container wins over any cloud
+	// setting for it.
+	for _, svc := range c.latest {
+		if svc.Key == serviceKey && svc.LabelIntent != nil && svc.LabelIntent.Logs == proto.LogModeOff {
+			return proto.LogModeOff
+		}
+	}
 	if m := c.logOverrides[serviceKey]; m != "" {
 		return m
 	}

cloud/labels.go

@@ -0,0 +1,155 @@
+package cloud
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/marvinvr/docktail/cloud/proto"
+	"github.com/marvinvr/docktail/docker"
+	apptypes "github.com/marvinvr/docktail/types"
+)
+
+// cloudLabels is what a container's docktail.cloud.* labels say, after
+// validation. Invalid values are dropped (never guessed at) and described in
+// problems so the collector can warn about them.
+type cloudLabels struct {
+	ignored  bool               // docktail.cloud.ignore=true
+	intent   *proto.LabelIntent // nil when no valid intent label is set
+	problems []string           // one line per dropped or unknown label, in label order
+}
+
+// parseCloudLabels validates a container's docktail.cloud.* labels. The same
+// bounds apply as to cloud-pushed config (proto.SanitizeLabelIntent), so a
+// label can never shape a check the cloud itself could not have sent.
+func parseCloudLabels(labels map[string]string) cloudLabels {
+	var out cloudLabels
+	if len(labels) == 0 {
+		return out
+	}
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var raw proto.LabelIntent
+	for _, key := range keys {
+		value := strings.TrimSpace(labels[key])
+		switch key {
+		case apptypes.LabelCloudIgnore:
+			switch strings.ToLower(value) {
+			case "true":
+				out.ignored = true
+			case "false":
+			default:
+				out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be true or false", key, value))
+			}
+		case apptypes.LabelCloudLogs:
+			if strings.EqualFold(value, proto.LogModeOff) {
+				raw.Logs = proto.LogModeOff
+			} else {
+				out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: the only value is off (capture itself is switched on in DockTail Cloud)", key, value))
+			}
+		case apptypes.LabelCloudCheckKind:
+			switch kind := strings.ToLower(value); kind {
+			case "tcp", "http":
+				raw.CheckKind = kind
+			default:
+				out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be tcp or http", key, value))
+			}
+		case apptypes.LabelCloudCheckPath:
+			if value == "" || proto.ValidateHTTPPath(value) != nil {
+				out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be a relative path starting with /", key, value))
+			} else {
+				raw.CheckPath = value
+			}
+		case apptypes.LabelCloudCheckExpectStatus:
+			code, err := strconv.Atoi(value)
+			if err != nil || code < 100 || code > 599 {
+				out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be an HTTP status code (100-599)", key, value))
+			} else {
+				raw.CheckExpectStatus = code
+			}
+		default:
+			out.problems = append(out.problems, fmt.Sprintf("%s: unknown DockTail Cloud label, ignored", key))
+		}
+	}
+	if raw.CheckKind == "tcp" && (raw.CheckPath != "" || raw.CheckExpectStatus != 0) {
+		out.problems = append(out.problems, fmt.Sprintf("%s and %s ignored: %s=tcp has no HTTP request",
+			apptypes.LabelCloudCheckPath, apptypes.LabelCloudCheckExpectStatus, apptypes.LabelCloudCheckKind))
+	}
+	out.intent, _ = proto.SanitizeLabelIntent(&raw)
+	return out
+}
+
+// applyLabelIntent lays a service's label intent over the cloud's check config
+// for it (base; nil when the cloud sent none). Each label wins for its own
+// field, a field with no label keeps the cloud's value, and a path or expected
+// status with no kind label implies http. ok is false when the labels say
+// nothing about the check (or the result would not validate): keep base.
+func applyLabelIntent(serviceKey string, base *proto.CheckConfig, intent *proto.LabelIntent) (proto.CheckConfig, bool) {
+	if intent == nil || (intent.CheckKind == "" && intent.CheckPath == "" && intent.CheckExpectStatus == 0) {
+		return proto.CheckConfig{}, false
+	}
+	cfg := proto.CheckConfig{Kind: "tcp", IntervalMS: proto.DefaultCheckIntervalMS}
+	if base != nil {
+		cfg = *base
+	}
+	cfg.ServiceKey = serviceKey
+	if cfg.IntervalMS == 0 {
+		cfg.IntervalMS = proto.DefaultCheckIntervalMS
+	}
+	switch {
+	case intent.CheckKind != "":
+		cfg.Kind = intent.CheckKind
+	case intent.CheckPath != "" || intent.CheckExpectStatus != 0:
+		cfg.Kind = "http"
+	}
+	if intent.CheckPath != "" {
+		cfg.Path = intent.CheckPath
+	}
+	if intent.CheckExpectStatus != 0 {
+		cfg.ExpectStatus = intent.CheckExpectStatus
+	}
+	if cfg.Kind == "tcp" {
+		cfg.Path = ""
+		cfg.ExpectStatus = 0
+	}
+	if proto.ValidateCheckConfig(cfg) != nil {
+		return proto.CheckConfig{}, false
+	}
+	return cfg, true
+}
+
+// intentForService narrows a container's label intent to one of the services
+// it publishes. Cloud labels are container-wide, but a service whose backend
+// speaks TCP (docktail.service[.N].protocol=tcp or tls-terminated-tcp) cannot
+// answer a plain HTTP check, so for it an HTTP-shaping label (kind http, a path, an expected
+// status) is dropped and it keeps its TCP check; logs=off still applies.
+func intentForService(intent *proto.LabelIntent, backendProtocol string) *proto.LabelIntent {
+	if p := strings.ToLower(backendProtocol); intent == nil || (p != "tcp" && p != "tls-terminated-tcp") {
+		return intent
+	}
+	out := *intent
+	if out.CheckKind == "http" {
+		out.CheckKind = ""
+	}
+	out.CheckPath = ""
+	out.CheckExpectStatus = 0
+	if out == (proto.LabelIntent{}) {
+		return nil
+	}
+	return &out
+}
+
+// labelsForbidCapture reports whether a container's labels (a docker event's
+// actor attributes carry them all) rule out incident log capture: the
+// container is ignored by DockTail Cloud, or has docktail.cloud.logs=off.
+func labelsForbidCapture(labels map[string]string) bool {
+	if docker.IsCloudIgnored(labels) {
+		return true
+	}
+	return strings.EqualFold(strings.TrimSpace(labels[apptypes.LabelCloudLogs]), proto.LogModeOff)
+}