From b988e0279149a0445ab0d1d7e3b05fa11708dab9 Mon Sep 17 00:00:00 2001 From: Marvin von Rappard Date: Wed, 23 Sep 2026 22:41:45 +0200 Subject: [PATCH] feat: add --version, a health check that tracks DockTail, and an update notice - A shared version package: release builds inject the version there, the startup log carries it, and `docktail --version` prints it. - `docktail health` reads a status file the running process rewrites every 10 s and becomes the image's HEALTHCHECK: unhealthy when DockTail is hung, the reconcile loop stalls, Docker stays unreachable, or the tailscaled socket is gone. Per-service reconcile errors and the DockTail Cloud link are reported without failing the check. Nothing listens on a port. - A daily update check (UPDATE_CHECK=false to disable) logs a newer stable release once. - The cloud module records its link state for the health status and warns when the host clock is more than 60 s off the cloud's. --- Dockerfile | 6 +- cloud/collector.go | 20 ++- cloud/link.go | 116 ++++++++++++++ docs/06-cloud.md | 8 + docs/07-reference.md | 70 +++++++++ e2e.sh | 14 ++ health/health.go | 323 +++++++++++++++++++++++++++++++++++++++ health/health_test.go | 71 +++++++++ main.go | 73 ++++++++- reconciler/reconciler.go | 28 +++- version/semver.go | 128 ++++++++++++++++ version/semver_test.go | 44 ++++++ version/update.go | 124 +++++++++++++++ version/version.go | 8 + 14 files changed, 1023 insertions(+), 10 deletions(-) create mode 100644 cloud/link.go create mode 100644 health/health.go create mode 100644 health/health_test.go create mode 100644 version/semver.go create mode 100644 version/semver_test.go create mode 100644 version/update.go create mode 100644 version/version.go diff --git a/Dockerfile b/Dockerfile index fe50e0f..58d1160 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ COPY . . # Build the application RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo \ - -ldflags "-w -s -X github.com/marvinvr/docktail/cloud.agentVersion=${VERSION}" \ + -ldflags "-w -s -X github.com/marvinvr/docktail/version.Version=${VERSION}" \ -o docktail . # Tailscale binary stage — ensures CLI version matches the sidecar daemon exactly @@ -36,7 +36,9 @@ WORKDIR /app # Copy binary from build stage COPY --from=builder /build/docktail . +# Reads the status file the running process keeps current: healthy while the +# reconcile loop keeps succeeding (see docs/07-reference.md#health-check). HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD tailscale --socket=${TAILSCALE_SOCKET:-/var/run/tailscale/tailscaled.sock} serve status || exit 1 + CMD ["/app/docktail", "health"] ENTRYPOINT ["/bin/sh", "-c", "sleep 1 && exec /app/docktail"] diff --git a/cloud/collector.go b/cloud/collector.go index d55a718..7b4daa7 100644 --- a/cloud/collector.go +++ b/cloud/collector.go @@ -15,11 +15,12 @@ import ( "github.com/marvinvr/docktail/cloud/proto" "github.com/marvinvr/docktail/docker" apptypes "github.com/marvinvr/docktail/types" + "github.com/marvinvr/docktail/version" ) -// agentVersion is reported in Hello. Release builds replace the development -// value with the DockTail image tag via -ldflags. -var agentVersion = "dev" +// agentVersion is reported in Hello: the DockTail build version, which release +// builds set via -ldflags on version.Version. +var agentVersion = version.Version // restartLoopThreshold is the container RestartCount above which a die is also // treated as a restart-loop signal. @@ -78,6 +79,9 @@ type Collector struct { hostTempCap bool // temperature sensors detected → advertise host_temp hostDiskCap bool // filesystems enumerable here → advertise host_disk loadNodeScoped bool // /proc loadavg is the physical node's, not this CT's → don't report it + + createdAt time.Time // when the collector was built; the link has been "connecting" since + link linkTracker // connection state for the local health status (link.go) } // cpuSample is the previous CPU counter reading kept per container. Docker @@ -131,6 +135,7 @@ func NewCollector(ctx context.Context, cfg Config, dc *docker.Client, ts tailnet hostMetricsCap: hmr.available(), hostTempCap: hmr.tempAvailable(), hostDiskCap: hmr.diskAvailable(), + createdAt: time.Now(), } // On a Proxmox LXC the agent's /proc is the physical node's, so loadavg is the // whole node's load — meaningless against the CT's (smaller) core count, where @@ -664,6 +669,7 @@ func (c *Collector) session(ctx context.Context, bo *backoff) (stop bool) { dialCtx, dialCancel := context.WithTimeout(ctx, 20*time.Second) conn, err := dial(dialCtx, c.cfg.URL, c.cfg.Key, c.log) dialCancel() + c.noteDial(err) if err != nil { var de *dialError if asDialError(err, &de) && (de.statusCode == 401 || de.statusCode == 403) { @@ -680,6 +686,7 @@ func (c *Collector) session(ctx context.Context, bo *backoff) (stop bool) { ackCh := make(chan proto.HelloAck, 1) h := handlers{ onHelloAck: func(ack proto.HelloAck) { + c.noteHelloAck(ack) select { case ackCh <- ack: default: @@ -1145,14 +1152,19 @@ func (c *Collector) setConn(conn *wsConn) { c.mu.Lock() c.conn = conn c.mu.Unlock() + c.setLink(LinkConnected, "") } func (c *Collector) clearConn(conn *wsConn) { c.mu.Lock() - if c.conn == conn { + cleared := c.conn == conn + if cleared { c.conn = nil } c.mu.Unlock() + if cleared { + c.setLink(LinkDisconnected, "connection closed") + } } func (c *Collector) applyConfig(cfg proto.Config) { diff --git a/cloud/link.go b/cloud/link.go new file mode 100644 index 0000000..74bd753 --- /dev/null +++ b/cloud/link.go @@ -0,0 +1,116 @@ +package cloud + +import ( + "fmt" + "sync" + "time" + + "github.com/marvinvr/docktail/cloud/proto" +) + +// Link states reported by Collector.LinkStatus, for DockTail's local health +// status (see ../health). +const ( + LinkConnecting = "connecting" // no connection accepted yet + LinkConnected = "connected" // an accepted connection is up + LinkDisconnected = "disconnected" // lost the connection or cannot reach the cloud; retrying + LinkRejected = "rejected" // the cloud refused the last attempt (Reason names why) +) + +// clockSkewThreshold is how far this host's clock may drift from the cloud's +// before the agent warns: timestamps the agent sends (docker events, check +// results) would otherwise land out of order with the cloud's own. +const clockSkewThreshold = 60 * time.Second + +// clockSkewWarnEvery throttles the skew warning across reconnects. +const clockSkewWarnEvery = time.Hour + +// LinkStatus is the cloud connection state at a point in time. +type LinkStatus struct { + State string + Since time.Time // when State was entered + Reason string // rejection reason or last connection error, if any +} + +// linkTracker records the link state. It has its own lock so reading it never +// waits on the collector's. +type linkTracker struct { + mu sync.Mutex + status LinkStatus + lastSkewWarn time.Time +} + +// LinkStatus reports the current cloud connection state. +func (c *Collector) LinkStatus() LinkStatus { + c.link.mu.Lock() + defer c.link.mu.Unlock() + if c.link.status.State == "" { + return LinkStatus{State: LinkConnecting, Since: c.createdAt} + } + return c.link.status +} + +func (c *Collector) setLink(state, reason string) { + c.link.mu.Lock() + defer c.link.mu.Unlock() + if c.link.status.State != state { + c.link.status.Since = time.Now() + } + c.link.status.State = state + c.link.status.Reason = reason +} + +// noteDial records the outcome of a dial that did not produce a connection. A +// 401/403 upgrade is the cloud (or something in front of it) refusing this +// agent; anything else is an ordinary failure that is retried. +func (c *Collector) noteDial(err error) { + if err == nil { + return + } + var de *dialError + if asDialError(err, &de) && (de.statusCode == 401 || de.statusCode == 403) { + c.setLink(LinkRejected, fmt.Sprintf("http_%d", de.statusCode)) + return + } + c.setLink(LinkDisconnected, err.Error()) +} + +// noteHelloAck records a hello_ack: a refusal sets the link to rejected, and an +// acceptance is checked for clock skew against the cloud's clock. +func (c *Collector) noteHelloAck(ack proto.HelloAck) { + if !ack.Accepted { + c.setLink(LinkRejected, string(ack.Reason)) + return + } + c.checkClockSkew(ack.ServerTime) +} + +// checkClockSkew warns when this host's clock is more than clockSkewThreshold +// away from the cloud's (serverMS is the hello_ack's server_time). The frame's +// transit time is ignored: it is far below the threshold. +func (c *Collector) checkClockSkew(serverMS int64) { + if serverMS <= 0 { + return // an older cloud that does not send server_time + } + now := time.Now() + skew := now.Sub(time.UnixMilli(serverMS)) + if skew > -clockSkewThreshold && skew < clockSkewThreshold { + return + } + c.link.mu.Lock() + throttled := !c.link.lastSkewWarn.IsZero() && now.Sub(c.link.lastSkewWarn) < clockSkewWarnEvery + if !throttled { + c.link.lastSkewWarn = now + } + c.link.mu.Unlock() + if throttled { + return + } + direction := "ahead of" + if skew < 0 { + direction = "behind" + } + c.log.Warn(). + Dur("skew", skew.Round(time.Second)). + Msgf("cloud: this host's clock is %s %s DockTail Cloud's; event and check times will be off. Enable time sync (NTP) on the host", skew.Abs().Round(time.Second), direction) +} diff --git a/docs/06-cloud.md b/docs/06-cloud.md index cd68ad3..abb806f 100644 --- a/docs/06-cloud.md +++ b/docs/06-cloud.md @@ -37,6 +37,14 @@ services: With no key set, no connection is opened and DockTail runs exactly as before. +`docktail health` reports the connection state (`connecting`, `connected`, +`disconnected`, `rejected` with the reason, or `failed` when the module could +not start, for example with a malformed key) without ever making the container unhealthy; see +[Health Check](07-reference.md#health-check). When the cloud accepts the +connection the agent also compares its clock with the cloud's and warns if they +are more than 60 seconds apart, since event and check times would then be off; +enable time sync (NTP) on the host. + If Cloud marks a host as unmonitored (for example, the host sits past the workspace's host cap), the agent keeps the connection open with heartbeats and occasional catalog snapshots and pauses checks, Docker events, metrics, tailnet diff --git a/docs/07-reference.md b/docs/07-reference.md index 6a05536..2a763c1 100644 --- a/docs/07-reference.md +++ b/docs/07-reference.md @@ -20,6 +20,8 @@ Use this section when checking exact configuration names, defaults, and supporte | `TAILSCALE_SOCKET` | `/var/run/tailscale/tailscaled.sock` | Tailscale daemon socket. | | `EXIT_ON_SOCKET_LOSS` | `true` | When `true`, DockTail exits if the Tailscale socket stays unreachable past the grace period, so the container's restart policy can re-establish the mount. See [Tailscale Socket Loss](#tailscale-socket-loss). | | `SOCKET_LOSS_GRACE_PERIOD` | `90s` | How long the Tailscale socket may stay unreachable before DockTail exits. Must be longer than a normal `tailscaled` restart. | +| `UPDATE_CHECK` | `true` | When `true`, DockTail checks once a day whether a newer release exists and logs it once. Set `false` to never contact GitHub. See [Version And Updates](#version-and-updates). | +| `HEALTH_FILE` | `/tmp/docktail-health.json` | Where DockTail writes the status file that `docktail health` (the image's health check) reads. Point it at a writable path if the container's `/tmp` is read-only. See [Health Check](#health-check). | If both OAuth and API key credentials are configured, DockTail uses OAuth. @@ -171,6 +173,74 @@ Prefer a named volume over a host path when you run `tailscaled` as a sidecar: a volume keeps one directory for its lifetime, so recreating the sidecar cannot detach DockTail's mount in the first place. +### Version And Updates + +DockTail logs its version on startup (`Starting DockTail version=…`), and the +binary prints it on request: + +```bash +docker exec docktail /app/docktail --version +# or, without a running container: +docker run --rm --entrypoint /app/docktail ghcr.io/marvinvr/docktail:latest --version +``` + +Include that version in bug reports. + +About 30 seconds after startup and then once a day (hourly after a failed +check), a release build asks the +GitHub API for the repository's tags and, when a newer stable release exists, +logs it once: + +```text +INF A newer DockTail release is available; pull the new image and recreate the container to update (set UPDATE_CHECK=false to stop checking) current=1.8.2 latest=1.8.3 release_notes=https://github.com/marvinvr/docktail/releases +``` + +The request is an anonymous `GET` to `api.github.com` that carries nothing +about your installation beyond the `docktail/` user agent (GitHub +sees the source IP, as with any request). It is +skipped for development builds, a failure is only logged at debug level, and +`UPDATE_CHECK=false` turns it off. Pre-release tags are never offered, and a +pre-release build newer than the latest stable release is not nagged. + +### Health Check + +The image's `HEALTHCHECK` runs `docktail health`, which tracks DockTail itself +rather than only the Tailscale daemon. The running process rewrites a small +JSON status file every 10 seconds (and after every reconcile) at `HEALTH_FILE`; +`docktail health` reads it, prints a one-line verdict, and exits `0` when +healthy and `1` when not. Nothing listens on a port. + +DockTail is **unhealthy** when: + +- the status file is missing or has not been updated for a minute (DockTail is + hung, stopped, or cannot write the file); +- the `tailscaled` socket does not accept connections; +- the last two reconciles could not read the containers from Docker; +- no reconcile has finished for three reconcile intervals (at least three + minutes), meaning the loop is stuck. + +A reconcile that fails for individual services — a label conflict, a service +the tailnet refuses — is reported as `healthy, with errors`: that is one +container's configuration, and DockTail keeps serving the rest. + +The [DockTail Cloud](06-cloud.md#docktail-cloud) link state (`connecting`, +`connected`, `disconnected`, `rejected`, or `failed`) is included in the output +but never makes the container unhealthy: a cloud outage or a revoked key does +not stop DockTail from serving containers, and a restart triggered by the +health status (Swarm, autoheal) would drain every service without fixing it. + +```bash +docker exec docktail /app/docktail health +# healthy: last reconcile 12s ago; cloud connected for 3h12m4s + +docker inspect --format '{{json .State.Health}}' docktail # Docker's view, with recent outputs +docker exec docktail cat /tmp/docktail-health.json # the full status +``` + +With a read-only root filesystem, mount a `tmpfs` at `/tmp` or set +`HEALTH_FILE` to a writable path; otherwise DockTail logs a warning and the +health check reports unhealthy. + ### Useful Links - Tailscale Services documentation: `https://tailscale.com/kb/1552/tailscale-services` diff --git a/e2e.sh b/e2e.sh index 92cc933..886b367 100755 --- a/e2e.sh +++ b/e2e.sh @@ -1244,6 +1244,20 @@ else pass "no FATAL or panic in logs" fi +# The image's HEALTHCHECK runs `docktail health`, which reads the status file +# the running process keeps current. +if health_out=$(docker exec "$DOCKTAIL_CONTAINER" /app/docktail health 2>&1); then + pass "docktail health reports healthy ($health_out)" +else + fail "docktail health reports unhealthy: $health_out" +fi + +if version_out=$(docker exec "$DOCKTAIL_CONTAINER" /app/docktail --version 2>&1) && [[ "$version_out" == docktail\ * ]]; then + pass "docktail --version prints the version ($version_out)" +else + fail "docktail --version failed: $version_out" +fi + # ============================================================================== # Summary # ============================================================================== diff --git a/health/health.go b/health/health.go new file mode 100644 index 0000000..fb6815b --- /dev/null +++ b/health/health.go @@ -0,0 +1,323 @@ +// Package health is DockTail's local status surface. The running process keeps +// a small JSON status file current (Tracker), and `docktail health` reads it +// back and turns it into a verdict (Check) — which is what the image's +// HEALTHCHECK runs. Nothing listens on a port: the status never leaves the +// container unless someone with access to it reads the file. +package health + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/rs/zerolog" +) + +// EnvFile overrides where the status file lives. Both the running process and +// `docktail health` read it, so they always agree inside one container. +const EnvFile = "HEALTH_FILE" + +const ( + // writeInterval is how often the status file is rewritten even when nothing + // changed; its updated_at is how Check tells a live process from a hung one. + writeInterval = 10 * time.Second + // staleAfter is how old updated_at may get before Check calls the process hung. + staleAfter = 60 * time.Second + // minReconcileBudget floors how long the reconcile loop may go without + // finishing a cycle before it counts as stalled; the budget is otherwise + // three intervals. + minReconcileBudget = 3 * time.Minute + // dockerFailureLimit is how many reconciles in a row may fail to read the + // containers from Docker before DockTail is unhealthy; one is a blip. + dockerFailureLimit = 2 +) + +// Cloud link states, as reported by the optional DockTail Cloud module. +const ( + CloudConnecting = "connecting" // enabled, no connection accepted yet + CloudConnected = "connected" // an accepted connection is up + CloudDisconnected = "disconnected" // retrying after a lost connection or failed dial + CloudRejected = "rejected" // the cloud refused the connection (see Reason) + CloudFailed = "failed" // the module could not start (see Reason) +) + +// Status is the content of the status file. +type Status struct { + Version string `json:"version"` + PID int `json:"pid"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` + ReconcileIntervalSeconds float64 `json:"reconcile_interval_seconds"` + Reconcile Reconcile `json:"reconcile"` + // Tailscale is nil when no socket probe is configured. + Tailscale *Tailscale `json:"tailscale,omitempty"` + // Cloud is nil when DockTail Cloud is not configured. + Cloud *Cloud `json:"cloud,omitempty"` +} + +// Reconcile is the outcome of the reconcile loop so far. +type Reconcile struct { + LastRunAt *time.Time `json:"last_run_at,omitempty"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + LastError string `json:"last_error,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures"` + // DockerFailures counts the latest reconciles in a row that could not read + // the containers from Docker. + DockerFailures int `json:"docker_failures"` +} + +// Tailscale is the reachability of the tailscaled socket. +type Tailscale struct { + SocketReachable bool `json:"socket_reachable"` + LastError string `json:"last_error,omitempty"` +} + +// Cloud is the DockTail Cloud link state. +type Cloud struct { + State string `json:"state"` + Since time.Time `json:"since"` + Reason string `json:"reason,omitempty"` +} + +// Path is the status file location: $HEALTH_FILE, else docktail-health.json in +// the temp directory (/tmp in the image). +func Path() string { + if p := strings.TrimSpace(os.Getenv(EnvFile)); p != "" { + return p + } + return filepath.Join(os.TempDir(), "docktail-health.json") +} + +// Tracker collects the running process's state and keeps the status file +// current. Its methods are safe for concurrent use. +type Tracker struct { + path string + log zerolog.Logger + nudge chan struct{} + + mu sync.Mutex + status Status + cloud func() *Cloud + tailscale func() error +} + +// NewTracker prepares a tracker; nothing is written until Run. +func NewTracker(path, version string, reconcileInterval time.Duration, logger zerolog.Logger) *Tracker { + return &Tracker{ + path: path, + log: logger, + nudge: make(chan struct{}, 1), + status: Status{ + Version: version, + PID: os.Getpid(), + StartedAt: time.Now().UTC(), + ReconcileIntervalSeconds: reconcileInterval.Seconds(), + }, + } +} + +// SetCloudSource installs the function that reports the cloud link state. Leave +// it unset when DockTail Cloud is not configured. +func (t *Tracker) SetCloudSource(fn func() *Cloud) { + t.mu.Lock() + t.cloud = fn + t.mu.Unlock() +} + +// SetTailscaleProbe installs the function that checks the tailscaled socket +// (nil error = reachable). It runs before every write. +func (t *Tracker) SetTailscaleProbe(fn func() error) { + t.mu.Lock() + t.tailscale = fn + t.mu.Unlock() +} + +// ReconcileDone records the outcome of one reconcile cycle. dockerUnreachable +// marks a failure to read the containers from Docker at all, as opposed to a +// failure while applying individual services. +func (t *Tracker) ReconcileDone(err error, dockerUnreachable bool) { + now := time.Now().UTC() + t.mu.Lock() + r := &t.status.Reconcile + r.LastRunAt = &now + if err == nil { + r.LastSuccessAt = &now + r.LastError = "" + r.ConsecutiveFailures = 0 + } else { + r.LastError = err.Error() + r.ConsecutiveFailures++ + } + if err != nil && dockerUnreachable { + r.DockerFailures++ + } else { + r.DockerFailures = 0 + } + t.mu.Unlock() + select { + case t.nudge <- struct{}{}: + default: + } +} + +// Run writes the status file now, after every reconcile and every +// writeInterval, until ctx ends; it then removes the file so nothing reads a +// stopped process as healthy. A failed write is logged once, not per attempt. +func (t *Tracker) Run(ctx context.Context) { + ticker := time.NewTicker(writeInterval) + defer ticker.Stop() + defer func() { _ = os.Remove(t.path) }() + failing := false + for { + if err := t.write(); err != nil { + if !failing { + t.log.Warn().Err(err).Str("path", t.path). + Msgf("Cannot write the health status file, so `docktail health` (the container healthcheck) will report unhealthy; point %s at a writable path", EnvFile) + } + failing = true + } else { + failing = false + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + case <-t.nudge: + } + } +} + +func (t *Tracker) snapshot() Status { + t.mu.Lock() + s := t.status + cloud, probe := t.cloud, t.tailscale + t.mu.Unlock() + if cloud != nil { + s.Cloud = cloud() + } + if probe != nil { + s.Tailscale = &Tailscale{SocketReachable: true} + if err := probe(); err != nil { + s.Tailscale = &Tailscale{LastError: err.Error()} + } + } + s.UpdatedAt = time.Now().UTC() + return s +} + +// write replaces the status file atomically (temp file + rename), so a +// concurrent `docktail health` never reads a half-written file. +func (t *Tracker) write() error { + data, err := json.MarshalIndent(t.snapshot(), "", " ") + if err != nil { + return err + } + dir := filepath.Dir(t.path) + tmp, err := os.CreateTemp(dir, ".docktail-health-*") + if err != nil { + return err + } + _, werr := tmp.Write(append(data, '\n')) + cerr := tmp.Close() + if werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Rename(tmp.Name(), t.path) + } + if werr != nil { + _ = os.Remove(tmp.Name()) + } + return werr +} + +// Check reads the status file at path and decides whether DockTail is healthy, +// returning a one-line explanation either way. +// +// Healthy means DockTail can do its job: the process is alive (the file was +// rewritten recently), the reconcile loop keeps finishing cycles (within three +// reconcile intervals, at least three minutes), Docker answers, and the +// tailscaled socket accepts connections. A cycle that fails on individual services (a +// label conflict, a service the tailnet refuses) is reported but stays healthy: +// that is one container's configuration, and DockTail keeps serving the rest. +// +// The DockTail Cloud link is reported but never decides the verdict: an outage +// or a rejection there does not stop DockTail from serving containers, and a +// restart triggered by an unhealthy status (Swarm, autoheal) would drain every +// service without fixing anything on the cloud side. +func Check(path string, now time.Time) (bool, string) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return false, fmt.Sprintf("unhealthy: no status file at %s (DockTail is not running, is still starting, or cannot write it; see %s)", path, EnvFile) + } + return false, fmt.Sprintf("unhealthy: cannot read %s: %v", path, err) + } + var s Status + if err := json.Unmarshal(data, &s); err != nil { + return false, fmt.Sprintf("unhealthy: cannot parse %s: %v", path, err) + } + if age := now.Sub(s.UpdatedAt); age > staleAfter { + return false, fmt.Sprintf("unhealthy: status not updated for %s (DockTail is hung or stopped)", roundAge(age)) + } + + budget := 3 * time.Duration(s.ReconcileIntervalSeconds*float64(time.Second)) + if budget < minReconcileBudget { + budget = minReconcileBudget + } + r := s.Reconcile + cloud := cloudSummary(s.Cloud, now) + + if s.Tailscale != nil && !s.Tailscale.SocketReachable { + return false, fmt.Sprintf("unhealthy: tailscaled socket unreachable: %s; %s", s.Tailscale.LastError, cloud) + } + if r.DockerFailures >= dockerFailureLimit { + return false, fmt.Sprintf("unhealthy: Docker unreachable in the last %d reconciles%s; %s", r.DockerFailures, lastErr(r), cloud) + } + if r.LastRunAt == nil { + if since := now.Sub(s.StartedAt); since > budget { + return false, fmt.Sprintf("unhealthy: no reconcile finished in %s since start (reconcile loop stalled); %s", roundAge(since), cloud) + } + return true, "healthy: starting, no reconcile finished yet; " + cloud + } + if age := now.Sub(*r.LastRunAt); age > budget { + return false, fmt.Sprintf("unhealthy: no reconcile finished in %s (reconcile loop stalled); %s", roundAge(age), cloud) + } + if r.ConsecutiveFailures > 0 { + return true, fmt.Sprintf("healthy, with errors: last %d reconcile(s) failed%s; %s", r.ConsecutiveFailures, lastErr(r), cloud) + } + return true, fmt.Sprintf("healthy: last reconcile %s ago; %s", roundAge(now.Sub(*r.LastRunAt)), cloud) +} + +func lastErr(r Reconcile) string { + if r.LastError == "" { + return "" + } + return " (last error: " + r.LastError + ")" +} + +func cloudSummary(c *Cloud, now time.Time) string { + if c == nil { + return "cloud not configured" + } + s := "cloud " + c.State + if !c.Since.IsZero() { + s += " for " + roundAge(now.Sub(c.Since)) + } + if c.Reason != "" { + s += " (" + c.Reason + ")" + } + return s +} + +func roundAge(d time.Duration) string { + if d < 0 { + d = 0 + } + return d.Round(time.Second).String() +} diff --git a/health/health_test.go b/health/health_test.go new file mode 100644 index 0000000..b07d90c --- /dev/null +++ b/health/health_test.go @@ -0,0 +1,71 @@ +package health + +import ( + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" +) + +func TestCheck(t *testing.T) { + path := filepath.Join(t.TempDir(), "status.json") + expect := func(name string, now time.Time, healthy bool, fragment string) { + t.Helper() + ok, msg := Check(path, now) + if ok != healthy || !strings.Contains(msg, fragment) { + t.Fatalf("%s: got %v %q, want %v containing %q", name, ok, msg, healthy, fragment) + } + } + + expect("missing file", time.Now(), false, "no status file") + + tr := NewTracker(path, "1.2.3", time.Minute, zerolog.Nop()) + write := func() { + t.Helper() + if err := tr.write(); err != nil { + t.Fatal(err) + } + } + write() + now := time.Now() + expect("fresh start", now, true, "cloud not configured") + expect("fresh start", now, true, "starting") + + // The loop never finished a cycle, long past the budget. + tr.status.StartedAt = now.Add(-10 * time.Minute) + write() + expect("stalled at start", now, false, "stalled") + + // One failure to reach Docker is a blip; two in a row are not. + tr.ReconcileDone(errors.New("docker unreachable"), true) + write() + expect("one docker failure", now, true, "docker unreachable") + tr.ReconcileDone(errors.New("docker unreachable"), true) + write() + expect("two docker failures", now, false, "Docker unreachable") + + // A per-service failure keeps DockTail healthy. + tr.ReconcileDone(errors.New("service configuration error"), false) + write() + expect("service error", now, true, "with errors") + + // The cloud link is reported, never decisive. + tr.ReconcileDone(nil, false) + tr.SetCloudSource(func() *Cloud { return &Cloud{State: CloudRejected, Since: now, Reason: "invalid_key"} }) + write() + expect("cloud rejected", now, true, "cloud rejected") + + // An unreachable tailscaled socket is unhealthy. + tr.SetTailscaleProbe(func() error { return errors.New("socket gone") }) + write() + expect("tailscale down", now, false, "socket gone") + tr.SetTailscaleProbe(func() error { return nil }) + write() + expect("tailscale back", now, true, "healthy") + + // A process that stopped rewriting the file is unhealthy. + expect("stale file", now.Add(2*time.Minute), false, "not updated") +} diff --git a/main.go b/main.go index 77872a3..f06cd33 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,9 @@ package main import ( "context" + "errors" + "fmt" + "io" "os" "os/signal" "strconv" @@ -16,15 +19,21 @@ import ( "github.com/marvinvr/docktail/cloud" "github.com/marvinvr/docktail/docker" + "github.com/marvinvr/docktail/health" "github.com/marvinvr/docktail/reconciler" "github.com/marvinvr/docktail/tailscale" + "github.com/marvinvr/docktail/version" ) func main() { + if len(os.Args) > 1 { + os.Exit(runCommand(os.Args[1:])) + } + // Setup logging setupLogging() - log.Info().Msg("Starting DockTail") + log.Info().Str("version", version.Version).Msg("Starting DockTail") // Get configuration from environment reconcileInterval := getEnvDuration("RECONCILE_INTERVAL", 60*time.Second) @@ -41,6 +50,11 @@ func main() { skipShutdownCleanup := getEnvBool("SKIP_SHUTDOWN_CLEANUP", false) exitOnSocketLoss := getEnvBool("EXIT_ON_SOCKET_LOSS", true) socketLossGracePeriod := getEnvDuration("SOCKET_LOSS_GRACE_PERIOD", 90*time.Second) + updateCheck := getEnvBool("UPDATE_CHECK", true) + healthFile := health.Path() + // A restarted container keeps /tmp: drop the previous run's status so it + // cannot vouch for this one before the tracker writes a fresh file. + _ = os.Remove(healthFile) // Parse default tags, dropping duplicates: the Control Plane stores tags // as a set, so a duplicated default would register as permanent drift and @@ -96,6 +110,8 @@ func main() { Bool("skip_shutdown_cleanup", skipShutdownCleanup). Bool("exit_on_socket_loss", exitOnSocketLoss). Dur("socket_loss_grace_period", socketLossGracePeriod). + Bool("update_check", updateCheck). + Str("health_file", healthFile). Msg("Configuration loaded") // Create Docker client @@ -131,6 +147,14 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // Local health status: a small JSON file `docktail health` (the image's + // HEALTHCHECK) reads back. Nothing listens on a port. + healthTracker := health.NewTracker(healthFile, version.Version, reconcileInterval, log.Logger) + healthTracker.SetTailscaleProbe(tailscaleClient.ProbeSocket) + rec.SetResultHook(func(err error) { + healthTracker.ReconcileDone(err, errors.Is(err, reconciler.ErrListContainers)) + }) + // Optional: DockTail Cloud reporting module. Completely inert unless // DOCKTAIL_CLOUD_KEY is set — DockTail runs exactly as before without it. if cloud.Enabled() { @@ -138,7 +162,15 @@ func main() { collector, cerr := cloud.NewCollector(ctx, cloudCfg, dockerClient, tailscaleClient, log.Logger) if cerr != nil { log.Error().Err(cerr).Msg("DockTail Cloud enabled but failed to initialize; continuing without it") + failedAt, reason := time.Now(), cerr.Error() + healthTracker.SetCloudSource(func() *health.Cloud { + return &health.Cloud{State: health.CloudFailed, Since: failedAt, Reason: reason} + }) } else { + healthTracker.SetCloudSource(func() *health.Cloud { + s := collector.LinkStatus() + return &health.Cloud{State: s.State, Since: s.Since, Reason: s.Reason} + }) rec.SetObserver(collector) go collector.Run(ctx) log.Info(). @@ -174,6 +206,10 @@ func main() { }, ) go watchdog.Run(ctx) + go healthTracker.Run(ctx) + if updateCheck { + go version.RunUpdateCheck(ctx, log.Logger) + } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) @@ -344,3 +380,38 @@ func logCredentialWarnings(tailscaleAPIKey, tailscaleOAuthClientID, tailscaleOAu event.Msg("Incomplete Tailscale OAuth configuration; both OAuth environment variables must be set to enable OAuth-based auto-service creation") } } + +// runCommand handles the command-line invocations; DockTail itself takes no +// arguments. It returns the process exit code. +func runCommand(args []string) int { + switch args[0] { + case "--version", "-version", "-v", "version": + fmt.Println("docktail " + version.Version) + return 0 + case "health", "healthcheck": + // Exit 0 when healthy and 1 otherwise, as Docker's HEALTHCHECK expects. + ok, summary := health.Check(health.Path(), time.Now()) + fmt.Println(summary) + if !ok { + return 1 + } + return 0 + case "--help", "-help", "-h", "help": + printUsage(os.Stdout) + return 0 + default: + _, _ = fmt.Fprintf(os.Stderr, "docktail: unknown argument %q\n\n", args[0]) + printUsage(os.Stderr) + return 2 + } +} + +func printUsage(w io.Writer) { + _, _ = fmt.Fprint(w, `Usage: + docktail run DockTail (configured through environment variables) + docktail --version print the version and exit + docktail health report whether the running DockTail is healthy (exit 0) or not (exit 1) + +Documentation: https://docktail.org/docs/ +`) +} diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 698138e..286e391 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -2,6 +2,7 @@ package reconciler import ( "context" + "errors" "fmt" "time" @@ -13,6 +14,11 @@ import ( apptypes "github.com/marvinvr/docktail/types" ) +// ErrListContainers marks a reconcile that failed before it could read the +// containers from Docker — Docker is unreachable, as opposed to a failure while +// applying one service. +var ErrListContainers = errors.New("failed to get enabled containers") + // Observer receives reconciler outputs for an optional side-channel consumer — // the cloud module (see ../cloud). The reconciler calls these inline, so an // implementation must return quickly (hand off to its own goroutines/queues). @@ -32,7 +38,14 @@ type Reconciler struct { dockerClient *docker.Client tailscaleClient *tailscale.Client interval time.Duration - observer Observer // optional; nil unless the cloud module is enabled + observer Observer // optional; nil unless the cloud module is enabled + onResult func(err error) // optional; told the outcome of every reconcile +} + +// SetResultHook installs a function told the outcome of every reconcile cycle +// (nil on success) — the local health status. Safe to call once before Run. +func (r *Reconciler) SetResultHook(fn func(err error)) { + r.onResult = fn } // SetObserver attaches an optional observer (the cloud collector). Pass nil to @@ -117,14 +130,23 @@ func (r *Reconciler) Run(ctx context.Context) error { } } -// Reconcile performs a single reconciliation cycle +// Reconcile performs a single reconciliation cycle and reports its outcome to +// the result hook, if one is set. func (r *Reconciler) Reconcile(ctx context.Context) error { + err := r.reconcile(ctx) + if r.onResult != nil { + r.onResult(err) + } + return err +} + +func (r *Reconciler) reconcile(ctx context.Context) error { log.Info().Msg("Starting reconciliation") // Get all enabled containers from Docker containers, err := r.dockerClient.GetEnabledContainers(ctx) if err != nil { - return fmt.Errorf("failed to get enabled containers: %w", err) + return fmt.Errorf("%w: %w", ErrListContainers, err) } log.Info(). diff --git a/version/semver.go b/version/semver.go new file mode 100644 index 0000000..2582394 --- /dev/null +++ b/version/semver.go @@ -0,0 +1,128 @@ +package version + +import ( + "strconv" + "strings" +) + +// Semver is a parsed semantic version: MAJOR.MINOR.PATCH with an optional +// pre-release. Build metadata is accepted and ignored, as it carries no +// precedence. +type Semver struct { + Major, Minor, Patch uint64 + Pre []string +} + +// Parse reads a semantic version, tolerating a leading "v". It reports false +// for anything else, including development builds ("dev", "latest"). +func Parse(s string) (Semver, bool) { + s = strings.TrimPrefix(strings.TrimSpace(s), "v") + if i := strings.IndexByte(s, '+'); i >= 0 { + s = s[:i] + } + core, pre, hasPre := strings.Cut(s, "-") + parts := strings.Split(core, ".") + if len(parts) != 3 { + return Semver{}, false + } + var nums [3]uint64 + for i, p := range parts { + if !isNumeric(p) { + return Semver{}, false + } + n, err := strconv.ParseUint(p, 10, 64) + if err != nil { + return Semver{}, false + } + nums[i] = n + } + v := Semver{Major: nums[0], Minor: nums[1], Patch: nums[2]} + if hasPre { + v.Pre = strings.Split(pre, ".") + for _, id := range v.Pre { + if id == "" { + return Semver{}, false + } + } + } + return v, true +} + +// Stable reports whether v is a final release rather than a pre-release. +func (v Semver) Stable() bool { return len(v.Pre) == 0 } + +// String renders v without a leading "v". +func (v Semver) String() string { + s := strconv.FormatUint(v.Major, 10) + "." + strconv.FormatUint(v.Minor, 10) + "." + strconv.FormatUint(v.Patch, 10) + if len(v.Pre) > 0 { + s += "-" + strings.Join(v.Pre, ".") + } + return s +} + +// Compare orders a and b by semver precedence: -1, 0 or +1. +func Compare(a, b Semver) int { + if c := cmpUint(a.Major, b.Major); c != 0 { + return c + } + if c := cmpUint(a.Minor, b.Minor); c != 0 { + return c + } + if c := cmpUint(a.Patch, b.Patch); c != 0 { + return c + } + // A release outranks any of its pre-releases. + switch { + case len(a.Pre) == 0 && len(b.Pre) == 0: + return 0 + case len(a.Pre) == 0: + return 1 + case len(b.Pre) == 0: + return -1 + } + for i := 0; i < len(a.Pre) && i < len(b.Pre); i++ { + if c := cmpIdent(a.Pre[i], b.Pre[i]); c != 0 { + return c + } + } + return cmpUint(uint64(len(a.Pre)), uint64(len(b.Pre))) +} + +// cmpIdent compares pre-release identifiers: numeric ones numerically and +// below alphanumeric ones, which compare as ASCII. +func cmpIdent(a, b string) int { + an, bn := isNumeric(a), isNumeric(b) + switch { + case an && bn: + ai, _ := strconv.ParseUint(a, 10, 64) + bi, _ := strconv.ParseUint(b, 10, 64) + return cmpUint(ai, bi) + case an: + return -1 + case bn: + return 1 + } + return strings.Compare(a, b) +} + +func cmpUint(a, b uint64) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + } + return 0 +} + +func isNumeric(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/version/semver_test.go b/version/semver_test.go new file mode 100644 index 0000000..580304f --- /dev/null +++ b/version/semver_test.go @@ -0,0 +1,44 @@ +package version + +import "testing" + +func TestParse(t *testing.T) { + valid := map[string]string{ + "1.8.3": "1.8.3", + "v1.8.3": "1.8.3", + "2.0.0-cloud.16": "2.0.0-cloud.16", + "1.2.3+build.5": "1.2.3", + } + for in, want := range valid { + v, ok := Parse(in) + if !ok || v.String() != want { + t.Errorf("Parse(%q) = %q, %v; want %q", in, v.String(), ok, want) + } + } + for _, in := range []string{"", "dev", "latest", "a.7.3", "1.8", "1.8.3.4", "1.8.x", "1.8.3-", "1.8.3-a..b"} { + if _, ok := Parse(in); ok { + t.Errorf("Parse(%q) accepted an invalid version", in) + } + } +} + +func TestCompare(t *testing.T) { + // Each version is lower than the next. + ordered := []string{ + "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-alpha.beta", "1.0.0-beta", + "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0-rc.1", "1.0.0", + "1.7.9", "1.8.0-debug.4", "1.8.0", "1.8.3", "2.0.0-cloud.2", "2.0.0-cloud.16", "2.0.0", + } + for i := 0; i+1 < len(ordered); i++ { + a, _ := Parse(ordered[i]) + b, _ := Parse(ordered[i+1]) + if Compare(a, b) != -1 || Compare(b, a) != 1 { + t.Errorf("expected %s < %s", ordered[i], ordered[i+1]) + } + } + a, _ := Parse("v1.8.3") + b, _ := Parse("1.8.3+meta") + if Compare(a, b) != 0 { + t.Errorf("expected v1.8.3 == 1.8.3+meta") + } +} diff --git a/version/update.go b/version/update.go new file mode 100644 index 0000000..c47a69d --- /dev/null +++ b/version/update.go @@ -0,0 +1,124 @@ +package version + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/rs/zerolog" +) + +// ReleasesURL is where the release notes live; the update notice points here. +const ReleasesURL = "https://github.com/marvinvr/docktail/releases" + +// tagsURL lists the repository's git tags. Every tag is published as an image +// tag by the release workflow, and every stable one also moves :latest, so the +// highest stable tag is the newest image a user can pull. +const tagsURL = "https://api.github.com/repos/marvinvr/docktail/tags" + +const ( + updateCheckDelay = 30 * time.Second // first check, after startup has settled + updateCheckInterval = 24 * time.Hour + updateCheckRetry = time.Hour // after a failed check + updateCheckTimeout = 15 * time.Second + maxTagPages = 10 // 100 tags a page +) + +// RunUpdateCheck looks for a newer stable DockTail release shortly after +// startup and then once a day (hourly after a failure), and logs each newer +// version it finds once. Each check is an anonymous GET to the GitHub API that +// sends nothing about this installation. Builds that are not a release version (dev, latest) skip it, +// and every failure is logged at debug level only. Blocks until ctx ends. +func RunUpdateCheck(ctx context.Context, logger zerolog.Logger) { + current, ok := Parse(Version) + if !ok { + logger.Debug().Str("version", Version).Msg("Update check skipped: not a release build") + return + } + client := &http.Client{Timeout: updateCheckTimeout} + var announced string + wait := updateCheckDelay + for { + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + latest, err := latestStable(ctx, client) + if err != nil { + logger.Debug().Err(err).Msg("Update check failed") + wait = updateCheckRetry + continue + } + wait = updateCheckInterval + if Compare(latest, current) <= 0 || latest.String() == announced { + continue + } + announced = latest.String() + logger.Info(). + Str("current", Version). + Str("latest", announced). + Str("release_notes", ReleasesURL). + Msg("A newer DockTail release is available; pull the new image and recreate the container to update (set UPDATE_CHECK=false to stop checking)") + } +} + +// latestStable returns the highest stable semver among the repository's tags. +func latestStable(ctx context.Context, client *http.Client) (Semver, error) { + var best Semver + found := false + for page := 1; page <= maxTagPages; page++ { + names, err := fetchTagPage(ctx, client, page) + if err != nil { + return Semver{}, err + } + for _, name := range names { + v, ok := Parse(name) + if !ok || !v.Stable() { + continue + } + if !found || Compare(v, best) > 0 { + best, found = v, true + } + } + if len(names) < 100 { + break + } + } + if !found { + return Semver{}, fmt.Errorf("no stable release tag found") + } + return best, nil +} + +func fetchTagPage(ctx context.Context, client *http.Client, page int) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, tagsURL+"?per_page=100&page="+strconv.Itoa(page), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("User-Agent", "docktail/"+Version) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GitHub API answered %s", resp.Status) + } + var tags []struct { + Name string `json:"name"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&tags); err != nil { + return nil, fmt.Errorf("decode tags: %w", err) + } + names := make([]string, len(tags)) + for i, t := range tags { + names[i] = t.Name + } + return names, nil +} diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000..fd52840 --- /dev/null +++ b/version/version.go @@ -0,0 +1,8 @@ +// Package version holds the DockTail build version and the optional check for +// a newer release. +package version + +// Version is the DockTail release this binary was built as. Release builds +// replace the development value with the image tag via -ldflags +// "-X github.com/marvinvr/docktail/version.Version=" (see the Dockerfile). +var Version = "dev"