From 3830358666d6306cdb17a26fb4f2ef67fc2a6a17 Mon Sep 17 00:00:00 2001 From: Marvin von Rappard Date: Wed, 23 Sep 2026 23:39:00 +0200 Subject: [PATCH 1/2] feat(cloud): docktail.cloud.* labels for monitoring intent Containers can now declare how DockTail Cloud monitors them, next to the rest of their labels: - docktail.cloud.ignore=true keeps the container out of Cloud monitoring: it is never reported as a service, checked or log-captured, and is listed in the container inventory marked label_ignored instead. - docktail.cloud.logs=off never captures incident log excerpts for it. - docktail.cloud.check.kind / .path / .expect-status shape the local check (a path or expected status implies http). The agent applies label intent itself: a label wins over the pushed config for its own setting, field by field, and settings without a label keep the cloud's value. Invalid values and unknown docktail.cloud.* keys are logged once per container and dropped, validated against the same bounds as cloud config, so a label can never pick a check destination. Valid intent is reported on each snapshot service (additive, omitempty proto fields) so the dashboard can show what is set by label. Nothing changes for the Tailscale reconciler. --- cloud/checks.go | 9 ++- cloud/collector.go | 92 ++++++++++++++++++++++-- cloud/labels.go | 155 ++++++++++++++++++++++++++++++++++++++++ cloud/labels_test.go | 129 +++++++++++++++++++++++++++++++++ cloud/proto/messages.go | 47 ++++++++++-- cloud/proto/schema.json | 16 ++++- cloud/proto/validate.go | 42 +++++++++++ docker/client.go | 46 ++++++++++++ docker/cloud.go | 42 ++++++----- docs/04-labels.md | 27 +++++++ docs/06-cloud.md | 7 +- types/types.go | 15 ++++ 12 files changed, 592 insertions(+), 35 deletions(-) create mode 100644 cloud/labels.go create mode 100644 cloud/labels_test.go diff --git a/cloud/checks.go b/cloud/checks.go index 9691432..d3622c2 100644 --- a/cloud/checks.go +++ b/cloud/checks.go @@ -46,8 +46,9 @@ type serviceCheck struct { } // run probes every service once. Cloud check config (keyed by service key) -// selects the check shape but never its destination; services with no locally -// discovered target are skipped. +// selects the check shape but never its destination, and the service's own +// docktail.cloud.check.* labels override it field by field; services with no +// locally discovered target are skipped. func (c *checker) run(ctx context.Context, services []proto.Service, configs []proto.CheckConfig) []proto.CheckResult { configs, _ = proto.SanitizeCheckConfigs(configs) cfgByKey := make(map[string]proto.CheckConfig, len(configs)) @@ -62,6 +63,10 @@ func (c *checker) run(ctx context.Context, services []proto.Service, configs []p cfg := cc sc.cfg = &cfg } + // docktail.cloud.check.* labels win over the cloud's config, field by field. + if merged, ok := applyLabelIntent(svc.Key, sc.cfg, svc.LabelIntent); ok { + sc.cfg = &merged + } if res, ok := c.runOne(ctx, sc); ok { results = append(results, res) } diff --git a/cloud/collector.go b/cloud/collector.go index d55a718..23bb795 100644 --- a/cloud/collector.go +++ b/cloud/collector.go @@ -55,6 +55,7 @@ type Collector struct { logMode string // workspace default capture mode ("" ⇒ proto.LogModeOff) logOverrides map[string]string // per-service capture mode override (service key -> proto.LogMode*) checkFails map[string]int // consecutive local-check failures per service key, for incident log capture + labelWarned map[string]string // container id -> the docktail.cloud.* label problems last warned about cfgVer int unmonitored bool // cloud reports this host inactive/past the plan cap; throttle output lastTeaser time.Time // last throttled teaser snapshot sent while unmonitored @@ -125,6 +126,7 @@ func NewCollector(ctx context.Context, cfg Config, dc *docker.Client, ts tailnet specs: specs, logOverrides: map[string]string{}, checkFails: map[string]int{}, + labelWarned: map[string]string{}, prevCPU: map[string]cpuSample{}, prevCPUOther: map[string]cpuSample{}, hostMx: hmr, @@ -153,7 +155,7 @@ func (c *Collector) Fingerprint() string { return c.fingerprint } // OnReconcile receives the reconciler's freshly computed services, enriches them // with runtime detail, stores them, and (if connected) sends a snapshot. func (c *Collector) OnReconcile(ctx context.Context, services []*apptypes.ContainerService) { - built := c.buildServices(ctx, services) + built := c.buildServices(ctx, services, false) c.mu.Lock() c.latest = built @@ -194,8 +196,14 @@ func (c *Collector) OnEvent(ctx context.Context, msg events.Message) { if unmonitored { return } + // The event's attributes carry the container's labels, so a label opt-out + // holds even before the container's first snapshot. + noCapture := labelsForbidCapture(msg.Actor.Attributes) for _, ev := range evs { c.sendOrSpool(conn, proto.TypeEvent, ev) + if noCapture { + continue + } // Capture the tail now, not at replay time: by the time the link is back the // container may have been recreated and its logs gone with it. c.maybeCaptureLogs(ctx, conn, ev) @@ -206,7 +214,11 @@ func (c *Collector) OnEvent(ctx context.Context, msg events.Message) { // buildServices maps reconciler ContainerService values to wire Services, // enriching each with a single inspect + stats sample per distinct container. -func (c *Collector) buildServices(ctx context.Context, services []*apptypes.ContainerService) []proto.Service { +// +// full marks a build from self-discovery, which lists every managed container +// (stopped ones included); only such a build may forget label warnings for +// containers it did not see. +func (c *Collector) buildServices(ctx context.Context, services []*apptypes.ContainerService, full bool) []proto.Service { type enriched struct { info docker.CloudInfo stats containerStats @@ -216,10 +228,21 @@ func (c *Collector) buildServices(ctx context.Context, services []*apptypes.Cont // of them, and toService stays a pure mapping. funnelHost := c.funnelHostname() out := make([]proto.Service, 0, len(services)) + labelled := make(map[string]struct{}) for _, cs := range services { if cs == nil { continue } + labels := parseCloudLabels(cs.CloudLabels) + if _, seen := labelled[cs.ContainerID]; !seen { + labelled[cs.ContainerID] = struct{}{} + c.warnLabelProblems(cs.ContainerID, cs.ContainerName, labels.problems) + } + if labels.ignored { + // docktail.cloud.ignore=true: not a cloud service at all. The container + // is reported as plain inventory (GetOtherContainers) instead. + continue + } e, ok := cache[cs.ContainerID] if !ok { if ci, err := c.docker.InspectCloud(ctx, cs.ContainerID); err == nil { @@ -228,16 +251,54 @@ func (c *Collector) buildServices(ctx context.Context, services []*apptypes.Cont e.stats = c.sampleStats(ctx, cs.ContainerID, e.info.State) cache[cs.ContainerID] = e } - out = append(out, toService(cs, e.info, e.stats, funnelHost)) + svc := toService(cs, e.info, e.stats, funnelHost) + svc.LabelIntent = intentForService(labels.intent, cs.Protocol) + out = append(out, svc) } present := make(map[string]struct{}, len(cache)) for id := range cache { present[id] = struct{}{} } c.pruneStats(present) + if full { + c.pruneLabelWarnings(labelled) + } return out } +// warnLabelProblems logs a container's invalid or unknown docktail.cloud.* +// labels once, and again only when the set of problems changes, so a bad label +// does not repeat on every discovery tick. +func (c *Collector) warnLabelProblems(containerID, containerName string, problems []string) { + signature := strings.Join(problems, "\n") + c.mu.Lock() + previous, known := c.labelWarned[containerID] + if signature == "" { + delete(c.labelWarned, containerID) + } else { + c.labelWarned[containerID] = signature + } + c.mu.Unlock() + if signature == "" || (known && previous == signature) { + return + } + for _, problem := range problems { + c.log.Warn().Str("container", containerName).Msg("cloud: " + problem) + } +} + +// pruneLabelWarnings forgets label warnings for containers no longer in the +// latest build, so a recreated container warns again and the map stays bounded. +func (c *Collector) pruneLabelWarnings(present map[string]struct{}) { + c.mu.Lock() + defer c.mu.Unlock() + for id := range c.labelWarned { + if _, ok := present[id]; !ok { + delete(c.labelWarned, id) + } + } +} + // sampleStats reads a one-shot docker stats sample for a running container and // turns it into a current-value usage reading. Memory is taken as-is; CPU% is // the delta of cumulative counters against this container's previous sample — @@ -453,6 +514,16 @@ func (c *Collector) eventBases(msg events.Message, attrs map[string]string) []pr } func (c *Collector) serviceKeysForEvent(msg events.Message, attrs map[string]string) []string { + // An ignored container is plain inventory to the cloud, so its events name + // the container, never a service — not even one the same-named container + // published before the label was added (compose recreates under the name). + if docker.IsCloudIgnored(attrs) { + if name := strings.TrimSpace(attrs["name"]); name != "" { + return []string{name} + } + return nil + } + c.mu.RLock() latest := c.latest c.mu.RUnlock() @@ -925,7 +996,7 @@ func (c *Collector) scanAndSnapshot(ctx context.Context, conn *wsConn) { c.log.Warn().Err(err).Msg("cloud: container discovery failed") return } - built := c.buildServices(ctx, containers) + built := c.buildServices(ctx, containers, true) c.mu.Lock() c.latest = built c.mu.Unlock() @@ -972,6 +1043,7 @@ func (c *Collector) buildContainers(ctx context.Context, containers []docker.Oth out = append(out, proto.Container{ ContainerID: oc.ID, IsAgent: oc.IsAgent, + LabelIgnored: oc.LabelIgnored, Name: oc.Name, Image: oc.Image, ImageTag: oc.ImageTag, @@ -1204,11 +1276,19 @@ func (c *Collector) applyConfig(cfg proto.Config) { Msg("cloud: applied config") } -// logModeFor returns the effective capture mode for a service key: its override -// when set, else the workspace default, else the built-in default (off). +// logModeFor returns the effective capture mode for a service key: off when its +// container is labelled docktail.cloud.logs=off, else its cloud override when +// set, else the workspace default, else the built-in default (off). func (c *Collector) logModeFor(serviceKey string) string { c.mu.RLock() defer c.mu.RUnlock() + // docktail.cloud.logs=off on the service's container wins over any cloud + // setting for it. + for _, svc := range c.latest { + if svc.Key == serviceKey && svc.LabelIntent != nil && svc.LabelIntent.Logs == proto.LogModeOff { + return proto.LogModeOff + } + } if m := c.logOverrides[serviceKey]; m != "" { return m } diff --git a/cloud/labels.go b/cloud/labels.go new file mode 100644 index 0000000..24e4bd5 --- /dev/null +++ b/cloud/labels.go @@ -0,0 +1,155 @@ +package cloud + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/marvinvr/docktail/cloud/proto" + "github.com/marvinvr/docktail/docker" + apptypes "github.com/marvinvr/docktail/types" +) + +// cloudLabels is what a container's docktail.cloud.* labels say, after +// validation. Invalid values are dropped (never guessed at) and described in +// problems so the collector can warn about them. +type cloudLabels struct { + ignored bool // docktail.cloud.ignore=true + intent *proto.LabelIntent // nil when no valid intent label is set + problems []string // one line per dropped or unknown label, in label order +} + +// parseCloudLabels validates a container's docktail.cloud.* labels. The same +// bounds apply as to cloud-pushed config (proto.SanitizeLabelIntent), so a +// label can never shape a check the cloud itself could not have sent. +func parseCloudLabels(labels map[string]string) cloudLabels { + var out cloudLabels + if len(labels) == 0 { + return out + } + keys := make([]string, 0, len(labels)) + for k := range labels { + keys = append(keys, k) + } + sort.Strings(keys) + + var raw proto.LabelIntent + for _, key := range keys { + value := strings.TrimSpace(labels[key]) + switch key { + case apptypes.LabelCloudIgnore: + switch strings.ToLower(value) { + case "true": + out.ignored = true + case "false": + default: + out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be true or false", key, value)) + } + case apptypes.LabelCloudLogs: + if strings.EqualFold(value, proto.LogModeOff) { + raw.Logs = proto.LogModeOff + } else { + out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: the only value is off (capture itself is switched on in DockTail Cloud)", key, value)) + } + case apptypes.LabelCloudCheckKind: + switch kind := strings.ToLower(value); kind { + case "tcp", "http": + raw.CheckKind = kind + default: + out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be tcp or http", key, value)) + } + case apptypes.LabelCloudCheckPath: + if value == "" || proto.ValidateHTTPPath(value) != nil { + out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be a relative path starting with /", key, value)) + } else { + raw.CheckPath = value + } + case apptypes.LabelCloudCheckExpectStatus: + code, err := strconv.Atoi(value) + if err != nil || code < 100 || code > 599 { + out.problems = append(out.problems, fmt.Sprintf("%s=%q ignored: must be an HTTP status code (100-599)", key, value)) + } else { + raw.CheckExpectStatus = code + } + default: + out.problems = append(out.problems, fmt.Sprintf("%s: unknown DockTail Cloud label, ignored", key)) + } + } + if raw.CheckKind == "tcp" && (raw.CheckPath != "" || raw.CheckExpectStatus != 0) { + out.problems = append(out.problems, fmt.Sprintf("%s and %s ignored: %s=tcp has no HTTP request", + apptypes.LabelCloudCheckPath, apptypes.LabelCloudCheckExpectStatus, apptypes.LabelCloudCheckKind)) + } + out.intent, _ = proto.SanitizeLabelIntent(&raw) + return out +} + +// applyLabelIntent lays a service's label intent over the cloud's check config +// for it (base; nil when the cloud sent none). Each label wins for its own +// field, a field with no label keeps the cloud's value, and a path or expected +// status with no kind label implies http. ok is false when the labels say +// nothing about the check (or the result would not validate): keep base. +func applyLabelIntent(serviceKey string, base *proto.CheckConfig, intent *proto.LabelIntent) (proto.CheckConfig, bool) { + if intent == nil || (intent.CheckKind == "" && intent.CheckPath == "" && intent.CheckExpectStatus == 0) { + return proto.CheckConfig{}, false + } + cfg := proto.CheckConfig{Kind: "tcp", IntervalMS: proto.DefaultCheckIntervalMS} + if base != nil { + cfg = *base + } + cfg.ServiceKey = serviceKey + if cfg.IntervalMS == 0 { + cfg.IntervalMS = proto.DefaultCheckIntervalMS + } + switch { + case intent.CheckKind != "": + cfg.Kind = intent.CheckKind + case intent.CheckPath != "" || intent.CheckExpectStatus != 0: + cfg.Kind = "http" + } + if intent.CheckPath != "" { + cfg.Path = intent.CheckPath + } + if intent.CheckExpectStatus != 0 { + cfg.ExpectStatus = intent.CheckExpectStatus + } + if cfg.Kind == "tcp" { + cfg.Path = "" + cfg.ExpectStatus = 0 + } + if proto.ValidateCheckConfig(cfg) != nil { + return proto.CheckConfig{}, false + } + return cfg, true +} + +// intentForService narrows a container's label intent to one of the services +// it publishes. Cloud labels are container-wide, but a service whose backend +// speaks raw TCP (docktail.service[.N].protocol=tcp) cannot answer an HTTP +// check, so for it an HTTP-shaping label (kind http, a path, an expected +// status) is dropped and it keeps its TCP check; logs=off still applies. +func intentForService(intent *proto.LabelIntent, backendProtocol string) *proto.LabelIntent { + if intent == nil || !strings.EqualFold(backendProtocol, "tcp") { + return intent + } + out := *intent + if out.CheckKind == "http" { + out.CheckKind = "" + } + out.CheckPath = "" + out.CheckExpectStatus = 0 + if out == (proto.LabelIntent{}) { + return nil + } + return &out +} + +// labelsForbidCapture reports whether a container's labels (a docker event's +// actor attributes carry them all) rule out incident log capture: the +// container is ignored by DockTail Cloud, or has docktail.cloud.logs=off. +func labelsForbidCapture(labels map[string]string) bool { + if docker.IsCloudIgnored(labels) { + return true + } + return strings.EqualFold(strings.TrimSpace(labels[apptypes.LabelCloudLogs]), proto.LogModeOff) +} diff --git a/cloud/labels_test.go b/cloud/labels_test.go new file mode 100644 index 0000000..e6cb6da --- /dev/null +++ b/cloud/labels_test.go @@ -0,0 +1,129 @@ +package cloud + +import ( + "testing" + + "github.com/marvinvr/docktail/cloud/proto" +) + +func TestParseCloudLabels(t *testing.T) { + tests := []struct { + name string + labels map[string]string + wantIgnored bool + wantIntent *proto.LabelIntent + wantProblems int + }{ + {name: "none", labels: nil}, + { + name: "ignore", + labels: map[string]string{"docktail.cloud.ignore": "True"}, + wantIgnored: true, + }, + { + name: "ignore false", + labels: map[string]string{"docktail.cloud.ignore": "false"}, + }, + { + name: "full intent", + labels: map[string]string{ + "docktail.cloud.logs": "off", + "docktail.cloud.check.kind": "HTTP", + "docktail.cloud.check.path": "/healthz?deep=1", + "docktail.cloud.check.expect-status": "204", + }, + wantIntent: &proto.LabelIntent{Logs: "off", CheckKind: "http", CheckPath: "/healthz?deep=1", CheckExpectStatus: 204}, + }, + { + name: "invalid values are dropped, valid ones kept", + labels: map[string]string{ + "docktail.cloud.logs": "incident", + "docktail.cloud.check.kind": "udp", + "docktail.cloud.check.path": "//evil.example/x", + "docktail.cloud.check.expect-status": "700", + "docktail.cloud.ignore": "yes", + "docktail.cloud.check.interval": "10s", + }, + wantProblems: 6, + }, + { + name: "tcp drops http-only fields", + labels: map[string]string{ + "docktail.cloud.check.kind": "tcp", + "docktail.cloud.check.path": "/healthz", + }, + wantIntent: &proto.LabelIntent{CheckKind: "tcp"}, + wantProblems: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseCloudLabels(tt.labels) + if got.ignored != tt.wantIgnored { + t.Errorf("ignored = %v, want %v", got.ignored, tt.wantIgnored) + } + if (got.intent == nil) != (tt.wantIntent == nil) || + (got.intent != nil && *got.intent != *tt.wantIntent) { + t.Errorf("intent = %+v, want %+v", got.intent, tt.wantIntent) + } + if len(got.problems) != tt.wantProblems { + t.Errorf("problems = %q, want %d", got.problems, tt.wantProblems) + } + }) + } +} + +func TestApplyLabelIntent(t *testing.T) { + cloudHTTP := &proto.CheckConfig{ServiceKey: "web:443", Kind: "http", Path: "/", ExpectStatus: 200, IntervalMS: 60_000} + + if _, ok := applyLabelIntent("web:443", cloudHTTP, nil); ok { + t.Fatal("no intent must keep the cloud config") + } + if _, ok := applyLabelIntent("web:443", cloudHTTP, &proto.LabelIntent{Logs: "off"}); ok { + t.Fatal("a logs-only intent says nothing about the check") + } + + // A path alone implies http over the TCP default. + got, ok := applyLabelIntent("web:443", nil, &proto.LabelIntent{CheckPath: "/healthz"}) + if !ok || got.Kind != "http" || got.Path != "/healthz" || got.IntervalMS != proto.DefaultCheckIntervalMS { + t.Fatalf("path only: got %+v ok=%v", got, ok) + } + + // Label wins field by field; unlabelled fields keep the cloud's values. + got, ok = applyLabelIntent("web:443", cloudHTTP, &proto.LabelIntent{CheckPath: "/ready"}) + if !ok || got.Kind != "http" || got.Path != "/ready" || got.ExpectStatus != 200 || got.IntervalMS != 60_000 { + t.Fatalf("merge: got %+v ok=%v", got, ok) + } + + // An explicit tcp label turns a cloud HTTP check back into TCP. + got, ok = applyLabelIntent("web:443", cloudHTTP, &proto.LabelIntent{CheckKind: "tcp"}) + if !ok || got.Kind != "tcp" || got.Path != "" || got.ExpectStatus != 0 { + t.Fatalf("tcp: got %+v ok=%v", got, ok) + } +} + +func TestLabelsForbidCapture(t *testing.T) { + if labelsForbidCapture(map[string]string{"name": "web"}) { + t.Fatal("no labels must not forbid capture") + } + if !labelsForbidCapture(map[string]string{"docktail.cloud.logs": "off"}) { + t.Fatal("logs=off must forbid capture") + } + if !labelsForbidCapture(map[string]string{"docktail.cloud.ignore": "true"}) { + t.Fatal("ignore=true must forbid capture") + } +} + +func TestIntentForService(t *testing.T) { + intent := &proto.LabelIntent{Logs: "off", CheckKind: "http", CheckPath: "/healthz"} + if got := intentForService(intent, "http"); got != intent { + t.Fatalf("http backend must keep the intent, got %+v", got) + } + got := intentForService(intent, "tcp") + if got == nil || *got != (proto.LabelIntent{Logs: "off"}) { + t.Fatalf("tcp backend: got %+v, want logs only", got) + } + if got := intentForService(&proto.LabelIntent{CheckPath: "/healthz"}, "tcp"); got != nil { + t.Fatalf("tcp backend with only HTTP intent: got %+v, want nil", got) + } +} diff --git a/cloud/proto/messages.go b/cloud/proto/messages.go index 6f568f9..d45572a 100644 --- a/cloud/proto/messages.go +++ b/cloud/proto/messages.go @@ -157,11 +157,46 @@ type Service struct { CPUPercent *float64 `json:"cpu_percent,omitempty"` // container CPU usage as % of all host cores MemUsageBytes int64 `json:"mem_usage_bytes,omitempty"` // working set (usage minus inactive file cache) MemLimitBytes int64 `json:"mem_limit_bytes,omitempty"` // effective limit (container limit, else host total) + + // LabelIntent is what the container's own docktail.cloud.* labels ask for + // this service. The agent has ALREADY applied it on top of [Config] (label + // wins for its own setting); the cloud receives it to know which settings + // are label-controlled. Nil ⇒ no valid cloud label. See [LabelIntent]. + LabelIntent *LabelIntent `json:"label_intent,omitempty"` +} + +// LabelIntent is the DockTail Cloud intent a container declares in its own +// labels, where the rest of its configuration lives. Every field is optional +// and names one label; an empty field means the label is absent (or its value +// was invalid and dropped by the agent). +// +// The agent is the one that applies it: for the service it overrides the +// matching field of the cloud-pushed [Config] — label wins for its own setting, +// and a setting with no label keeps the cloud's value. The cloud never echoes +// label intent back in a Config; it only records it (validated with +// [SanitizeLabelIntent]) to show which settings are set by label, and to keep +// its own server-side gates no looser than the label. +// +// - Logs (docktail.cloud.logs) is only ever [LogModeOff]: a label can switch +// incident log capture off for the service, never on. Enabling capture stays +// a workspace decision. +// - CheckKind / CheckPath / CheckExpectStatus (docktail.cloud.check.kind, +// .path, .expect-status) shape the local check exactly like the matching +// [CheckConfig] fields, under the same bounds. A path or expected status +// without a kind implies "http"; with kind "tcp" the agent drops them. +// +// docktail.cloud.ignore=true is not carried here: an ignored container is not a +// service at all, and is reported in [Containers] with LabelIgnored set. +type LabelIntent struct { + Logs string `json:"logs,omitempty"` // LogModeOff only + CheckKind string `json:"check_kind,omitempty"` // tcp/http + CheckPath string `json:"check_path,omitempty"` // bounded relative HTTP path (see ValidateHTTPPath) + CheckExpectStatus int `json:"check_expect_status,omitempty"` // 100–599 } // Containers is the inventory of NON-docktail containers the agent sees on the -// host — every running/stopped container that is not published as a docktail -// service. Unlike [Snapshot] (the monitored service catalog), these carry only +// host — every running/stopped container that is not reported as a docktail +// [Service], including one labelled docktail.cloud.ignore=true. Unlike [Snapshot] (the monitored service catalog), these carry only // descriptive, read-only metadata: there are no checks, vantages, or incidents // for them, so the cloud never alerts on them. Like Snapshot, a Full message is // authoritative for presence — the cloud upserts the listed containers and @@ -177,8 +212,12 @@ type Containers struct { // read-only metadata only — no exec/deploy surface. Identity is the docker // ContainerID (stable within a host). type Container struct { - ContainerID string `json:"container_id"` // docker container id (short) — identity within host - IsAgent bool `json:"is_agent,omitempty"` // true for the container running this reporting agent + ContainerID string `json:"container_id"` // docker container id (short) — identity within host + IsAgent bool `json:"is_agent,omitempty"` // true for the container running this reporting agent + // LabelIgnored marks a container labelled docktail.cloud.ignore=true. It is + // never reported as a [Service] (even if it publishes one), and the cloud + // never monitors it: it cannot be watched. + LabelIgnored bool `json:"label_ignored,omitempty"` Name string `json:"name"` Image string `json:"image"` ImageTag string `json:"image_tag,omitempty"` diff --git a/cloud/proto/schema.json b/cloud/proto/schema.json index 7709187..6c8f813 100644 --- a/cloud/proto/schema.json +++ b/cloud/proto/schema.json @@ -69,12 +69,23 @@ "restart_count": { "type": "integer" }, "cpu_percent": { "type": "number", "description": "live container CPU usage as % of all host cores" }, "mem_usage_bytes": { "type": "integer", "description": "live working-set memory (usage minus inactive file cache)" }, - "mem_limit_bytes": { "type": "integer", "description": "effective memory limit (container limit, else host total)" } + "mem_limit_bytes": { "type": "integer", "description": "effective memory limit (container limit, else host total)" }, + "label_intent": { "$ref": "#/$defs/LabelIntent" } + } + }, + "LabelIntent": { + "type": "object", + "description": "The container's own docktail.cloud.* labels, already applied by the agent over the cloud config (label wins for its own setting). Reported so the cloud can show what is label-controlled.", + "properties": { + "logs": { "type": "string", "enum": ["off"], "description": "docktail.cloud.logs — a label can only switch incident log capture off" }, + "check_kind": { "type": "string", "enum": ["tcp", "http"], "description": "docktail.cloud.check.kind" }, + "check_path": { "type": "string", "description": "docktail.cloud.check.path — bounded relative HTTP path" }, + "check_expect_status": { "type": "integer", "minimum": 100, "maximum": 599, "description": "docktail.cloud.check.expect-status" } } }, "Containers": { "type": "object", - "description": "Inventory of non-docktail containers (metadata only; no checks/incidents).", + "description": "Inventory of containers not reported as services, including docktail.cloud.ignore=true ones (metadata only; no checks).", "required": ["containers"], "properties": { "containers": { "type": "array", "items": { "$ref": "#/$defs/Container" } }, @@ -87,6 +98,7 @@ "properties": { "container_id": { "type": "string", "description": "docker container id (short); identity within host" }, "is_agent": { "type": "boolean", "description": "true when this container is the reporting DockTail agent" }, + "label_ignored": { "type": "boolean", "description": "labelled docktail.cloud.ignore=true: never a service, never monitored" }, "name": { "type": "string" }, "image": { "type": "string" }, "image_tag": { "type": "string" }, diff --git a/cloud/proto/validate.go b/cloud/proto/validate.go index 9d611ae..eceff8c 100644 --- a/cloud/proto/validate.go +++ b/cloud/proto/validate.go @@ -123,3 +123,45 @@ func containsControl(value string) bool { } return false } + +// SanitizeLabelIntent returns intent with every invalid field cleared, or nil +// when nothing valid remains; rejected counts the cleared fields. The agent +// applies it to what it parsed from labels, and the cloud again to what it +// received, so neither trusts the other's copy. A path or expected status +// beside CheckKind "tcp" is meaningless and is cleared too. +func SanitizeLabelIntent(intent *LabelIntent) (valid *LabelIntent, rejected int) { + if intent == nil { + return nil, 0 + } + out := *intent + if out.Logs != "" && out.Logs != LogModeOff { + out.Logs = "" + rejected++ + } + if out.CheckKind != "" && out.CheckKind != "tcp" && out.CheckKind != "http" { + out.CheckKind = "" + rejected++ + } + if out.CheckPath != "" && ValidateHTTPPath(out.CheckPath) != nil { + out.CheckPath = "" + rejected++ + } + if out.CheckExpectStatus != 0 && (out.CheckExpectStatus < 100 || out.CheckExpectStatus > 599) { + out.CheckExpectStatus = 0 + rejected++ + } + if out.CheckKind == "tcp" { + if out.CheckPath != "" { + out.CheckPath = "" + rejected++ + } + if out.CheckExpectStatus != 0 { + out.CheckExpectStatus = 0 + rejected++ + } + } + if out == (LabelIntent{}) { + return nil, rejected + } + return &out, rejected +} diff --git a/docker/client.go b/docker/client.go index 8abab33..44c56b4 100644 --- a/docker/client.go +++ b/docker/client.go @@ -99,6 +99,44 @@ func isFunnelEnabled(labels map[string]string) bool { return labels[apptypes.LabelFunnelEnable] == "true" } +// IsCloudIgnored reports whether a container opts out of DockTail Cloud with +// docktail.cloud.ignore=true. Such a container is never reported to the cloud +// as a service; it only affects cloud reporting, never the reconciler. +func IsCloudIgnored(labels map[string]string) bool { + return strings.EqualFold(strings.TrimSpace(labels[apptypes.LabelCloudIgnore]), "true") +} + +// cloudLabels returns the docktail.cloud.* subset of a container's labels, or +// nil when there is none. +func cloudLabels(labels map[string]string) map[string]string { + var out map[string]string + for k, v := range labels { + if !strings.HasPrefix(k, apptypes.LabelCloudPrefix) { + continue + } + if out == nil { + out = make(map[string]string) + } + out[k] = v + } + return out +} + +// withCloudLabels attaches the container's docktail.cloud.* labels to every +// service parsed from it. +func withCloudLabels(services []*apptypes.ContainerService, labels map[string]string) []*apptypes.ContainerService { + cl := cloudLabels(labels) + if cl == nil { + return services + } + for _, svc := range services { + if svc != nil { + svc.CloudLabels = cl + } + } + return services +} + func isManagedContainer(labels map[string]string) bool { return isServiceEnabled(labels) || isFunnelEnabled(labels) } @@ -746,6 +784,14 @@ func parseTags(tagsStr string, containerName string, defaultTags []string) []str } func (c *Client) parseContainer(ctx context.Context, containerID string, labels map[string]string) ([]*apptypes.ContainerService, error) { + services, err := c.parseContainerServices(ctx, containerID, labels) + if err != nil { + return nil, err + } + return withCloudLabels(services, labels), nil +} + +func (c *Client) parseContainerServices(ctx context.Context, containerID string, labels map[string]string) ([]*apptypes.ContainerService, error) { serviceEnabled := isServiceEnabled(labels) funnelEnabled := isFunnelEnabled(labels) if !serviceEnabled && !funnelEnabled { diff --git a/docker/cloud.go b/docker/cloud.go index 55185f9..8c625c6 100644 --- a/docker/cloud.go +++ b/docker/cloud.go @@ -55,7 +55,9 @@ func (c *Client) GetCloudContainers(ctx context.Context) ([]*apptypes.ContainerS var services []*apptypes.ContainerService for _, cont := range containers { - if !isManagedContainer(cont.Labels) { + // docktail.cloud.ignore=true: still served on the tailnet, but never a + // cloud service — GetOtherContainers reports it as plain inventory. + if !isManagedContainer(cont.Labels) || IsCloudIgnored(cont.Labels) { continue } @@ -77,7 +79,7 @@ func (c *Client) GetCloudContainers(ctx context.Context) ([]*apptypes.ContainerS log.Warn().Err(perr).Str("container", name).Msg("cloud: failed to parse running container, skipping") continue } - services = append(services, c.stoppedCloudServices(cont.ID, name, cont.Labels)...) + services = append(services, withCloudLabels(c.stoppedCloudServices(cont.ID, name, cont.Labels), cont.Labels)...) } return services, nil } @@ -89,6 +91,7 @@ func (c *Client) GetCloudContainers(ctx context.Context) ([]*apptypes.ContainerS type OtherContainer struct { ID string // short container id — identity within the host IsAgent bool // this container is running the reporting DockTail agent + LabelIgnored bool // labelled docktail.cloud.ignore=true (never a service, never monitored) Name string Image string ImageTag string @@ -101,11 +104,12 @@ type OtherContainer struct { CreatedAt int64 // unix seconds the container was created } -// GetOtherContainers lists every container that is NOT a docktail-managed -// service (neither docktail.enable nor docktail.funnel.enable set), INCLUDING -// stopped ones, for the cloud's container-inventory view. It is read-only and -// builds each entry straight from the container-list summary — no per-container -// inspect — so it stays cheap even on a busy host. Used only by the cloud module +// GetOtherContainers lists every container that is NOT reported as a docktail +// service — unmanaged (neither docktail.service.enable nor docktail.funnel.enable +// set) or labelled docktail.cloud.ignore=true — INCLUDING stopped ones, for the +// cloud's container-inventory view. It is read-only and builds each entry +// straight from the container-list summary — no per-container inspect — so it +// stays cheap even on a busy host. Used only by the cloud module // (DOCKTAIL_CLOUD_KEY set); docktail-managed containers are reported separately // by GetCloudContainers as services. func (c *Client) GetOtherContainers(ctx context.Context) ([]OtherContainer, error) { @@ -117,7 +121,8 @@ func (c *Client) GetOtherContainers(ctx context.Context) ([]OtherContainer, erro selfID := ownContainerID() out := make([]OtherContainer, 0, len(containers)) for _, cont := range containers { - if isManagedContainer(cont.Labels) { + ignored := IsCloudIgnored(cont.Labels) + if isManagedContainer(cont.Labels) && !ignored { continue // a docktail service — reported via GetCloudContainers } name := "" @@ -126,16 +131,17 @@ func (c *Client) GetOtherContainers(ctx context.Context) ([]OtherContainer, erro } image, tag := splitImageTag(cont.Image) oc := OtherContainer{ - ID: shortContainerID(cont.ID), - IsAgent: selfID != "" && cont.ID == selfID, - Name: name, - Image: image, - ImageTag: tag, - State: cont.State, - Status: cont.Status, - Health: healthFromStatus(cont.Status), - Ports: formatContainerPorts(cont.Ports), - CreatedAt: cont.Created, + ID: shortContainerID(cont.ID), + IsAgent: selfID != "" && cont.ID == selfID, + LabelIgnored: ignored, + Name: name, + Image: image, + ImageTag: tag, + State: cont.State, + Status: cont.Status, + Health: healthFromStatus(cont.Status), + Ports: formatContainerPorts(cont.Ports), + CreatedAt: cont.Created, } if cont.Labels != nil { oc.ComposeProject = cont.Labels["com.docker.compose.project"] diff --git a/docs/04-labels.md b/docs/04-labels.md index b39273a..43d7175 100644 --- a/docs/04-labels.md +++ b/docs/04-labels.md @@ -115,3 +115,30 @@ Funnel notes: - Funnel URLs use the machine hostname, not the Tailscale service name. - Funnel-only containers can omit `docktail.service.enable` and other `docktail.service.*` labels. - `docktail.service.direct` and `docktail.service.network` still control how DockTail reaches the backend for Funnel traffic. + +### DockTail Cloud Labels + +With [DockTail Cloud](06-cloud.md) enabled, `docktail.cloud.*` labels declare how Cloud monitors a container, next to the rest of its configuration. They change nothing about what DockTail serves on your tailnet, and are ignored when Cloud is not enabled. + +| Label | Values | Description | +| --- | --- | --- | +| `docktail.cloud.ignore` | `true` / `false` | `true` keeps the container out of Cloud monitoring: it is not reported as a service, is never checked, captures no logs, and cannot be watched. It still appears in the host's container inventory, marked as ignored by label, and its Docker events (start, stop, exit) still show in the activity log like any other container's. | +| `docktail.cloud.logs` | `off` | Never capture incident log excerpts for this container. Capture can only be switched *on* in the Cloud dashboard. | +| `docktail.cloud.check.kind` | `tcp` / `http` | Kind of local check. Without the label, the dashboard's setting applies, else `tcp`. | +| `docktail.cloud.check.path` | `/path` | Path the HTTP check requests. Must start with `/`. Implies `check.kind=http`. | +| `docktail.cloud.check.expect-status` | `100`–`599` | The only HTTP status that counts as up. Without it, any status below 500 is up. Implies `check.kind=http`. | + +```yaml +labels: + - "docktail.service.enable=true" + - "docktail.service.name=api" + - "docktail.service.port=8000" + - "docktail.cloud.check.path=/healthz" + - "docktail.cloud.logs=off" +``` + +A label wins over the dashboard for the setting it names: while `docktail.cloud.logs=off` is set, the dashboard shows that service's log capture as set by label and dashboard changes to it do not apply. Settings without a label keep their dashboard value. + +The check always targets the port DockTail already probes for the service; labels choose only how it is checked, never where. Cloud labels apply to every service a container publishes, including [numbered services](#multiple-services-from-one-container) — except that a service whose backend protocol is `tcp` keeps its TCP check whatever the HTTP check labels say. An invalid value (for example `docktail.cloud.check.kind=udp`) or an unknown `docktail.cloud.*` label is logged as a warning and ignored. `check.path` and `check.expect-status` are ignored when `check.kind=tcp`. + +Adding `docktail.cloud.ignore=true` to a container Cloud already monitors reads as that service being removed from the catalog; removing the label brings it back. diff --git a/docs/06-cloud.md b/docs/06-cloud.md index cd68ad3..81dd972 100644 --- a/docs/06-cloud.md +++ b/docs/06-cloud.md @@ -64,13 +64,14 @@ The hosted control plane (the dashboard and ingest service behind `wss://ingest. When enabled, the agent reports the following operational data: - Periodic snapshots of DockTail-managed services, including stopped containers, plus refreshes after successful reconciles. -- A read-only inventory of the host's **other** containers — the ones *not* published with `docktail.*` labels, including stopped ones — with name, image, state/health, ports, and live CPU/memory. These containers are not actively probed; they are listed on the dashboard so you can see the host's whole Docker footprint, and can be explicitly watched for Docker-event-driven incidents and alerts. +- A read-only inventory of the host's **other** containers — the ones *not* published with `docktail.*` labels, plus any labelled `docktail.cloud.ignore=true`, including stopped ones — with name, image, state/health, ports, and live CPU/memory. These containers are not actively probed; they are listed on the dashboard so you can see the host's whole Docker footprint, and can be explicitly watched for Docker-event-driven incidents and alerts. - Docker failure events, including container exit codes, out-of-memory (OOM) kills, health-status changes, and restart loops. -- Local-vantage check results. Checks default to TCP; cloud-managed config may select HTTP, a relative path, and expected status, but the destination always comes from the agent's local service discovery. +- Local-vantage check results. Checks default to TCP; cloud-managed config or the container's own [`docktail.cloud.check.*` labels](04-labels.md#docktail-cloud-labels) may select HTTP, a relative path, and expected status, but the destination always comes from the agent's local service discovery. +- Each service's valid `docktail.cloud.*` label values, so the dashboard can show which settings are set by label. - Whole-host vitals, sampled every 30 seconds: CPU, memory and swap usage, load average, temperature where the machine has sensors, and per-filesystem disk usage (mount point, total, used, and available bytes, at most 16 filesystems). These describe the machine, not the containers; nothing is stored as history, each report replaces the last. Disk is read on Linux only, from `/proc/mounts` and `statfs`, and covers real filesystems only — network mounts such as NFS and CIFS are deliberately skipped, so an unresponsive NAS can never stall reporting. - Tailscale control-plane service state, when Cloud asks for it (see [Tailnet Health](#tailnet-health)). - For a Funnel-exposed service, this node's MagicDNS name — the public address its Funnel answers on (see [Public Health](#public-health)). -- Bounded incident log excerpts. Cloud enables this by default and you can turn it off for the whole workspace or for an individual service; the agent captures nothing while the mode is off. Before sending, the agent best-effort redacts common Authorization/Bearer credentials, passwords, tokens, API keys, credential URLs, JWTs, and private-key blocks, then applies the 40-line/8-KiB caps. Redaction cannot recognize every application-specific secret, so turn capture off if your logs carry secrets those patterns will not match. +- Bounded incident log excerpts. Cloud enables this by default and you can turn it off for the whole workspace or for an individual service, in the dashboard or with the `docktail.cloud.logs=off` label; the agent captures nothing while the mode is off. Before sending, the agent best-effort redacts common Authorization/Bearer credentials, passwords, tokens, API keys, credential URLs, JWTs, and private-key blocks, then applies the 40-line/8-KiB caps. Redaction cannot recognize every application-specific secret, so turn capture off if your logs carry secrets those patterns will not match. If the connection to Cloud drops, the agent keeps watching Docker and buffers the failure events it sees — plus the log tail for each, captured at the moment of the diff --git a/types/types.go b/types/types.go index 93d0a57..1e35e00 100644 --- a/types/types.go +++ b/types/types.go @@ -36,6 +36,10 @@ type ContainerService struct { FunnelFunnelPort string // Public-facing port (443, 8443, or 10000 for HTTPS) FunnelProtocol string // Funnel protocol (https, tcp, tls-terminated-tcp) FunnelPath string // HTTP(S) Funnel path (for example "/" or "/webhook") + // CloudLabels holds the container's docktail.cloud.* labels verbatim (nil + // when it has none). Only the optional DockTail Cloud module reads them; the + // Tailscale reconciler ignores them. + CloudLabels map[string]string } // TailscaleServiceConfig represents the JSON structure for Tailscale service configuration @@ -69,3 +73,14 @@ const ( LabelDirect = "docktail.service.direct" // Direct container IP proxying (default: true, set to "false" to use published ports) LabelNetwork = "docktail.service.network" // Docker network to use for container IP (default: bridge or first available) ) + +// DockTail Cloud labels. They only matter with DOCKTAIL_CLOUD_KEY set, and +// never change what the reconciler serves on the tailnet. +const ( + LabelCloudPrefix = "docktail.cloud." + LabelCloudIgnore = "docktail.cloud.ignore" // "true": Cloud never monitors this container + LabelCloudLogs = "docktail.cloud.logs" // "off": never capture incident log excerpts + LabelCloudCheckKind = "docktail.cloud.check.kind" // local check kind: tcp or http + LabelCloudCheckPath = "docktail.cloud.check.path" // HTTP check path (implies kind http) + LabelCloudCheckExpectStatus = "docktail.cloud.check.expect-status" // HTTP status that counts as up (implies kind http) +) From 6301db9cc59ea059a74f9da85a46663e88fd4432 Mon Sep 17 00:00:00 2001 From: Marvin von Rappard Date: Wed, 23 Sep 2026 23:42:21 +0200 Subject: [PATCH 2/2] cloud: keep tls-terminated-tcp backends on their TCP check too --- cloud/labels.go | 6 +++--- cloud/labels_test.go | 2 +- docs/04-labels.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cloud/labels.go b/cloud/labels.go index 24e4bd5..fa73b48 100644 --- a/cloud/labels.go +++ b/cloud/labels.go @@ -125,11 +125,11 @@ func applyLabelIntent(serviceKey string, base *proto.CheckConfig, intent *proto. // intentForService narrows a container's label intent to one of the services // it publishes. Cloud labels are container-wide, but a service whose backend -// speaks raw TCP (docktail.service[.N].protocol=tcp) cannot answer an HTTP -// check, so for it an HTTP-shaping label (kind http, a path, an expected +// speaks TCP (docktail.service[.N].protocol=tcp or tls-terminated-tcp) cannot +// answer a plain HTTP check, so for it an HTTP-shaping label (kind http, a path, an expected // status) is dropped and it keeps its TCP check; logs=off still applies. func intentForService(intent *proto.LabelIntent, backendProtocol string) *proto.LabelIntent { - if intent == nil || !strings.EqualFold(backendProtocol, "tcp") { + if p := strings.ToLower(backendProtocol); intent == nil || (p != "tcp" && p != "tls-terminated-tcp") { return intent } out := *intent diff --git a/cloud/labels_test.go b/cloud/labels_test.go index e6cb6da..1625918 100644 --- a/cloud/labels_test.go +++ b/cloud/labels_test.go @@ -123,7 +123,7 @@ func TestIntentForService(t *testing.T) { if got == nil || *got != (proto.LabelIntent{Logs: "off"}) { t.Fatalf("tcp backend: got %+v, want logs only", got) } - if got := intentForService(&proto.LabelIntent{CheckPath: "/healthz"}, "tcp"); got != nil { + if got := intentForService(&proto.LabelIntent{CheckPath: "/healthz"}, "tls-terminated-tcp"); got != nil { t.Fatalf("tcp backend with only HTTP intent: got %+v, want nil", got) } } diff --git a/docs/04-labels.md b/docs/04-labels.md index 43d7175..754c289 100644 --- a/docs/04-labels.md +++ b/docs/04-labels.md @@ -139,6 +139,6 @@ labels: A label wins over the dashboard for the setting it names: while `docktail.cloud.logs=off` is set, the dashboard shows that service's log capture as set by label and dashboard changes to it do not apply. Settings without a label keep their dashboard value. -The check always targets the port DockTail already probes for the service; labels choose only how it is checked, never where. Cloud labels apply to every service a container publishes, including [numbered services](#multiple-services-from-one-container) — except that a service whose backend protocol is `tcp` keeps its TCP check whatever the HTTP check labels say. An invalid value (for example `docktail.cloud.check.kind=udp`) or an unknown `docktail.cloud.*` label is logged as a warning and ignored. `check.path` and `check.expect-status` are ignored when `check.kind=tcp`. +The check always targets the port DockTail already probes for the service; labels choose only how it is checked, never where. Cloud labels apply to every service a container publishes, including [numbered services](#multiple-services-from-one-container) — except that a service whose backend protocol is `tcp` or `tls-terminated-tcp` keeps its TCP check whatever the HTTP check labels say. An invalid value (for example `docktail.cloud.check.kind=udp`) or an unknown `docktail.cloud.*` label is logged as a warning and ignored. `check.path` and `check.expect-status` are ignored when `check.kind=tcp`. Adding `docktail.cloud.ignore=true` to a container Cloud already monitors reads as that service being removed from the catalog; removing the label brings it back.