feat: add --version, a health check that tracks DockTail, and an update notice

Dockerfile

@@ -17,7 +17,7 @@ COPY . .
 
 # Build the application
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo \
-    -ldflags "-w -s -X github.com/marvinvr/docktail/cloud.agentVersion=${VERSION}" \
+    -ldflags "-w -s -X github.com/marvinvr/docktail/version.Version=${VERSION}" \
     -o docktail .
 
 # Tailscale binary stage — ensures CLI version matches the sidecar daemon exactly
@@ -36,7 +36,9 @@ WORKDIR /app
 # Copy binary from build stage
 COPY --from=builder /build/docktail .
 
+# Reads the status file the running process keeps current: healthy while the
+# reconcile loop keeps succeeding (see docs/07-reference.md#health-check).
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
-  CMD tailscale --socket=${TAILSCALE_SOCKET:-/var/run/tailscale/tailscaled.sock} serve status || exit 1
+  CMD ["/app/docktail", "health"]
 
 ENTRYPOINT ["/bin/sh", "-c", "sleep 1 && exec /app/docktail"]

cloud/collector.go

@@ -15,11 +15,12 @@ import (
 	"github.com/marvinvr/docktail/cloud/proto"
 	"github.com/marvinvr/docktail/docker"
 	apptypes "github.com/marvinvr/docktail/types"
+	"github.com/marvinvr/docktail/version"
 )
 
-// agentVersion is reported in Hello. Release builds replace the development
-// value with the DockTail image tag via -ldflags.
-var agentVersion = "dev"
+// agentVersion is reported in Hello: the DockTail build version, which release
+// builds set via -ldflags on version.Version.
+var agentVersion = version.Version
 
 // restartLoopThreshold is the container RestartCount above which a die is also
 // treated as a restart-loop signal.
@@ -78,6 +79,9 @@ type Collector struct {
 	hostTempCap    bool               // temperature sensors detected → advertise host_temp
 	hostDiskCap    bool               // filesystems enumerable here → advertise host_disk
 	loadNodeScoped bool               // /proc loadavg is the physical node's, not this CT's → don't report it
+
+	createdAt time.Time   // when the collector was built; the link has been "connecting" since
+	link      linkTracker // connection state for the local health status (link.go)
 }
 
 // cpuSample is the previous CPU counter reading kept per container. Docker
@@ -131,6 +135,7 @@ func NewCollector(ctx context.Context, cfg Config, dc *docker.Client, ts tailnet
 		hostMetricsCap: hmr.available(),
 		hostTempCap:    hmr.tempAvailable(),
 		hostDiskCap:    hmr.diskAvailable(),
+		createdAt:      time.Now(),
 	}
 	// On a Proxmox LXC the agent's /proc is the physical node's, so loadavg is the
 	// whole node's load — meaningless against the CT's (smaller) core count, where
@@ -664,6 +669,7 @@ func (c *Collector) session(ctx context.Context, bo *backoff) (stop bool) {
 	dialCtx, dialCancel := context.WithTimeout(ctx, 20*time.Second)
 	conn, err := dial(dialCtx, c.cfg.URL, c.cfg.Key, c.log)
 	dialCancel()
+	c.noteDial(err)
 	if err != nil {
 		var de *dialError
 		if asDialError(err, &de) && (de.statusCode == 401 || de.statusCode == 403) {
@@ -680,6 +686,7 @@ func (c *Collector) session(ctx context.Context, bo *backoff) (stop bool) {
 	ackCh := make(chan proto.HelloAck, 1)
 	h := handlers{
 		onHelloAck: func(ack proto.HelloAck) {
+			c.noteHelloAck(ack)
 			select {
 			case ackCh <- ack:
 			default:
@@ -1145,14 +1152,19 @@ func (c *Collector) setConn(conn *wsConn) {
 	c.mu.Lock()
 	c.conn = conn
 	c.mu.Unlock()
+	c.setLink(LinkConnected, "")
 }
 
 func (c *Collector) clearConn(conn *wsConn) {
 	c.mu.Lock()
-	if c.conn == conn {
+	cleared := c.conn == conn
+	if cleared {
 		c.conn = nil
 	}
 	c.mu.Unlock()
+	if cleared {
+		c.setLink(LinkDisconnected, "connection closed")
+	}
 }
 
 func (c *Collector) applyConfig(cfg proto.Config) {

cloud/link.go

@@ -0,0 +1,116 @@
+package cloud
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/marvinvr/docktail/cloud/proto"
+)
+
+// Link states reported by Collector.LinkStatus, for DockTail's local health
+// status (see ../health).
+const (
+	LinkConnecting   = "connecting"   // no connection accepted yet
+	LinkConnected    = "connected"    // an accepted connection is up
+	LinkDisconnected = "disconnected" // lost the connection or cannot reach the cloud; retrying
+	LinkRejected     = "rejected"     // the cloud refused the last attempt (Reason names why)
+)
+
+// clockSkewThreshold is how far this host's clock may drift from the cloud's
+// before the agent warns: timestamps the agent sends (docker events, check
+// results) would otherwise land out of order with the cloud's own.
+const clockSkewThreshold = 60 * time.Second
+
+// clockSkewWarnEvery throttles the skew warning across reconnects.
+const clockSkewWarnEvery = time.Hour
+
+// LinkStatus is the cloud connection state at a point in time.
+type LinkStatus struct {
+	State  string
+	Since  time.Time // when State was entered
+	Reason string    // rejection reason or last connection error, if any
+}
+
+// linkTracker records the link state. It has its own lock so reading it never
+// waits on the collector's.
+type linkTracker struct {
+	mu           sync.Mutex
+	status       LinkStatus
+	lastSkewWarn time.Time
+}
+
+// LinkStatus reports the current cloud connection state.
+func (c *Collector) LinkStatus() LinkStatus {
+	c.link.mu.Lock()
+	defer c.link.mu.Unlock()
+	if c.link.status.State == "" {
+		return LinkStatus{State: LinkConnecting, Since: c.createdAt}
+	}
+	return c.link.status
+}
+
+func (c *Collector) setLink(state, reason string) {
+	c.link.mu.Lock()
+	defer c.link.mu.Unlock()
+	if c.link.status.State != state {
+		c.link.status.Since = time.Now()
+	}
+	c.link.status.State = state
+	c.link.status.Reason = reason
+}
+
+// noteDial records the outcome of a dial that did not produce a connection. A
+// 401/403 upgrade is the cloud (or something in front of it) refusing this
+// agent; anything else is an ordinary failure that is retried.
+func (c *Collector) noteDial(err error) {
+	if err == nil {
+		return
+	}
+	var de *dialError
+	if asDialError(err, &de) && (de.statusCode == 401 || de.statusCode == 403) {
+		c.setLink(LinkRejected, fmt.Sprintf("http_%d", de.statusCode))
+		return
+	}
+	c.setLink(LinkDisconnected, err.Error())
+}
+
+// noteHelloAck records a hello_ack: a refusal sets the link to rejected, and an
+// acceptance is checked for clock skew against the cloud's clock.
+func (c *Collector) noteHelloAck(ack proto.HelloAck) {
+	if !ack.Accepted {
+		c.setLink(LinkRejected, string(ack.Reason))
+		return
+	}
+	c.checkClockSkew(ack.ServerTime)
+}
+
+// checkClockSkew warns when this host's clock is more than clockSkewThreshold
+// away from the cloud's (serverMS is the hello_ack's server_time). The frame's
+// transit time is ignored: it is far below the threshold.
+func (c *Collector) checkClockSkew(serverMS int64) {
+	if serverMS <= 0 {
+		return // an older cloud that does not send server_time
+	}
+	now := time.Now()
+	skew := now.Sub(time.UnixMilli(serverMS))
+	if skew > -clockSkewThreshold && skew < clockSkewThreshold {
+		return
+	}
+	c.link.mu.Lock()
+	throttled := !c.link.lastSkewWarn.IsZero() && now.Sub(c.link.lastSkewWarn) < clockSkewWarnEvery
+	if !throttled {
+		c.link.lastSkewWarn = now
+	}
+	c.link.mu.Unlock()
+	if throttled {
+		return
+	}
+	direction := "ahead of"
+	if skew < 0 {
+		direction = "behind"
+	}
+	c.log.Warn().
+		Dur("skew", skew.Round(time.Second)).
+		Msgf("cloud: this host's clock is %s %s DockTail Cloud's; event and check times will be off. Enable time sync (NTP) on the host", skew.Abs().Round(time.Second), direction)
+}

docs/06-cloud.md

@@ -37,6 +37,14 @@ services:
 
 With no key set, no connection is opened and DockTail runs exactly as before.
 
+`docktail health` reports the connection state (`connecting`, `connected`,
+`disconnected`, `rejected` with the reason, or `failed` when the module could
+not start, for example with a malformed key) without ever making the container unhealthy; see
+[Health Check](07-reference.md#health-check). When the cloud accepts the
+connection the agent also compares its clock with the cloud's and warns if they
+are more than 60 seconds apart, since event and check times would then be off;
+enable time sync (NTP) on the host.
+
 If Cloud marks a host as unmonitored (for example, the host sits past the
 workspace's host cap), the agent keeps the connection open with heartbeats and
 occasional catalog snapshots and pauses checks, Docker events, metrics, tailnet

docs/07-reference.md

@@ -20,6 +20,8 @@ Use this section when checking exact configuration names, defaults, and supporte
 | `TAILSCALE_SOCKET` | `/var/run/tailscale/tailscaled.sock` | Tailscale daemon socket. |
 | `EXIT_ON_SOCKET_LOSS` | `true` | When `true`, DockTail exits if the Tailscale socket stays unreachable past the grace period, so the container's restart policy can re-establish the mount. See [Tailscale Socket Loss](#tailscale-socket-loss). |
 | `SOCKET_LOSS_GRACE_PERIOD` | `90s` | How long the Tailscale socket may stay unreachable before DockTail exits. Must be longer than a normal `tailscaled` restart. |
+| `UPDATE_CHECK` | `true` | When `true`, DockTail checks once a day whether a newer release exists and logs it once. Set `false` to never contact GitHub. See [Version And Updates](#version-and-updates). |
+| `HEALTH_FILE` | `/tmp/docktail-health.json` | Where DockTail writes the status file that `docktail health` (the image's health check) reads. Point it at a writable path if the container's `/tmp` is read-only. See [Health Check](#health-check). |
 
 If both OAuth and API key credentials are configured, DockTail uses OAuth.
 
@@ -171,6 +173,74 @@ Prefer a named volume over a host path when you run `tailscaled` as a sidecar: a
 volume keeps one directory for its lifetime, so recreating the sidecar cannot
 detach DockTail's mount in the first place.
 
+### Version And Updates
+
+DockTail logs its version on startup (`Starting DockTail version=…`), and the
+binary prints it on request:
+
+```bash
+docker exec docktail /app/docktail --version
+# or, without a running container:
+docker run --rm --entrypoint /app/docktail ghcr.io/marvinvr/docktail:latest --version
+```
+
+Include that version in bug reports.
+
+About 30 seconds after startup and then once a day (hourly after a failed
+check), a release build asks the
+GitHub API for the repository's tags and, when a newer stable release exists,
+logs it once:
+
+```text
+INF A newer DockTail release is available; pull the new image and recreate the container to update (set UPDATE_CHECK=false to stop checking) current=1.8.2 latest=1.8.3 release_notes=https://github.com/marvinvr/docktail/releases
+```
+
+The request is an anonymous `GET` to `api.github.com` that carries nothing
+about your installation beyond the `docktail/<version>` user agent (GitHub
+sees the source IP, as with any request). It is
+skipped for development builds, a failure is only logged at debug level, and
+`UPDATE_CHECK=false` turns it off. Pre-release tags are never offered, and a
+pre-release build newer than the latest stable release is not nagged.
+
+### Health Check
+
+The image's `HEALTHCHECK` runs `docktail health`, which tracks DockTail itself
+rather than only the Tailscale daemon. The running process rewrites a small
+JSON status file every 10 seconds (and after every reconcile) at `HEALTH_FILE`;
+`docktail health` reads it, prints a one-line verdict, and exits `0` when
+healthy and `1` when not. Nothing listens on a port.
+
+DockTail is **unhealthy** when:
+
+- the status file is missing or has not been updated for a minute (DockTail is
+  hung, stopped, or cannot write the file);
+- the `tailscaled` socket does not accept connections;
+- the last two reconciles could not read the containers from Docker;
+- no reconcile has finished for three reconcile intervals (at least three
+  minutes), meaning the loop is stuck.
+
+A reconcile that fails for individual services — a label conflict, a service
+the tailnet refuses — is reported as `healthy, with errors`: that is one
+container's configuration, and DockTail keeps serving the rest.
+
+The [DockTail Cloud](06-cloud.md#docktail-cloud) link state (`connecting`,
+`connected`, `disconnected`, `rejected`, or `failed`) is included in the output
+but never makes the container unhealthy: a cloud outage or a revoked key does
+not stop DockTail from serving containers, and a restart triggered by the
+health status (Swarm, autoheal) would drain every service without fixing it.
+
+```bash
+docker exec docktail /app/docktail health
+# healthy: last reconcile 12s ago; cloud connected for 3h12m4s
+
+docker inspect --format '{{json .State.Health}}' docktail   # Docker's view, with recent outputs
+docker exec docktail cat /tmp/docktail-health.json          # the full status
+```
+
+With a read-only root filesystem, mount a `tmpfs` at `/tmp` or set
+`HEALTH_FILE` to a writable path; otherwise DockTail logs a warning and the
+health check reports unhealthy.
+
 ### Useful Links
 
 - Tailscale Services documentation: `https://tailscale.com/kb/1552/tailscale-services`

e2e.sh

@@ -1244,6 +1244,20 @@ else
     pass "no FATAL or panic in logs"
 fi
 
+# The image's HEALTHCHECK runs `docktail health`, which reads the status file
+# the running process keeps current.
+if health_out=$(docker exec "$DOCKTAIL_CONTAINER" /app/docktail health 2>&1); then
+    pass "docktail health reports healthy ($health_out)"
+else
+    fail "docktail health reports unhealthy: $health_out"
+fi
+
+if version_out=$(docker exec "$DOCKTAIL_CONTAINER" /app/docktail --version 2>&1) && [[ "$version_out" == docktail\ * ]]; then
+    pass "docktail --version prints the version ($version_out)"
+else
+    fail "docktail --version failed: $version_out"
+fi
+
 # ==============================================================================
 # Summary
 # ==============================================================================