From d2bba63486871d6ba944d0e824e617d3087b034e Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 11:57:42 +0200 Subject: [PATCH 01/29] feat(config): add telemetry config block --- config/types.go | 10 ++++++++++ config/types_test.go | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 config/types_test.go diff --git a/config/types.go b/config/types.go index f631740d..58887269 100644 --- a/config/types.go +++ b/config/types.go @@ -16,6 +16,7 @@ type GitteConfig struct { GroupIncludes map[string][]string `yaml:"groupIncludes,omitempty"` Projects map[string]ProjectConfig `yaml:"projects,omitempty"` QuickSolve QuickSolveConfig `yaml:"quickSolve,omitempty"` + Telemetry TelemetryConfig `yaml:"telemetry,omitempty"` } // QuickSolveConfig holds settings for the quick solve feature in the actions TUI. @@ -28,6 +29,15 @@ type QuickSolveGitClean struct { Exclude []string `yaml:"exclude,omitempty"` } +// TelemetryConfig configures OpenTelemetry tracing export. Telemetry is enabled +// when an endpoint is resolved (from this config or the GITTE_TELEMETRY_URL env +// var). Headers carries arbitrary export headers, e.g. an Elastic APM secret +// token as Authorization: "Bearer " or an API key as "ApiKey ". +type TelemetryConfig struct { + Endpoint string `yaml:"endpoint,omitempty"` + Headers map[string]string `yaml:"headers,omitempty"` +} + // Template is a reusable project configuration template. // Extends lists parent template names (resolved left-to-right, self applied last). type Template struct { diff --git a/config/types_test.go b/config/types_test.go new file mode 100644 index 00000000..02ac1aae --- /dev/null +++ b/config/types_test.go @@ -0,0 +1,22 @@ +package config + +import "testing" + +func TestGitteConfig_TelemetryUnmarshal(t *testing.T) { + yamlData := []byte(` +telemetry: + endpoint: https://apm.example.com:8200 + headers: + Authorization: "Bearer secret" +`) + cfg, err := LoadGitteConfigFromYAML(yamlData) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Telemetry.Endpoint != "https://apm.example.com:8200" { + t.Errorf("endpoint = %q, want https://apm.example.com:8200", cfg.Telemetry.Endpoint) + } + if got := cfg.Telemetry.Headers["Authorization"]; got != "Bearer secret" { + t.Errorf("Authorization header = %q, want %q", got, "Bearer secret") + } +} From 135cea9ce15c0417bbfeadd0d62a47bc5e4a2dd4 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 12:00:50 +0200 Subject: [PATCH 02/29] feat(telemetry): add OTEL tracing core with OTLP/HTTP exporter --- go.mod | 25 ++++++- go.sum | 70 +++++++++++++++++-- telemetry/telemetry.go | 130 ++++++++++++++++++++++++++++++++++++ telemetry/telemetry_test.go | 81 ++++++++++++++++++++++ 4 files changed, 296 insertions(+), 10 deletions(-) create mode 100644 telemetry/telemetry.go create mode 100644 telemetry/telemetry_test.go diff --git a/go.mod b/go.mod index 07bf703c..6a6d2441 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,17 @@ require ( github.com/samber/lo v1.53.0 github.com/spf13/cobra v1.10.2 github.com/zalando/go-keyring v0.2.8 - golang.org/x/term v0.42.0 + go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 + golang.org/x/term v0.43.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20260413211237-bd52878bcec2 // indirect github.com/charmbracelet/x/ansi v0.11.7 // indirect @@ -25,7 +31,11 @@ require ( github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/danieljoos/wincred v1.2.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/mattn/go-runewidth v0.0.23 // indirect @@ -33,7 +43,16 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 7932f2f8..f8c8172f 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/ultraviolet v0.0.0-20260413211237-bd52878bcec2 h1:mRAlb/WARLaCnCwAEBa8Zfk965GrYc414MhJamV4anw= @@ -29,14 +33,31 @@ github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMF github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= @@ -47,6 +68,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= @@ -62,18 +85,51 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go new file mode 100644 index 00000000..26fa395e --- /dev/null +++ b/telemetry/telemetry.go @@ -0,0 +1,130 @@ +// Package telemetry wires OpenTelemetry tracing for gitte and exports spans to +// an OTLP/HTTP endpoint (e.g. Elastic APM). It is config-driven and degrades to +// a no-op whenever telemetry is disabled or setup fails, so it never blocks or +// slows gitte. +package telemetry + +import ( + "context" + "os" + "runtime" + "strings" + "time" + + "github.com/cego/gitte/config" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +const tracerName = "github.com/cego/gitte" + +const flushTimeout = 3 * time.Second + +// Resolved is the outcome of resolving telemetry settings from config + env. +type Resolved struct { + Enabled bool + Endpoint string // explicit endpoint to export to; empty when UseSDKEnv + Headers map[string]string // export headers (auth, etc.) + UseSDKEnv bool // enable from standard OTEL_* env; let the SDK read its own config +} + +// Resolve computes telemetry settings. Precedence: +// GITTE_TELEMETRY=off > GITTE_TELEMETRY_URL > config endpoint > OTEL_EXPORTER_OTLP_* env. +func Resolve(cfg *config.GitteConfig) Resolved { + if strings.EqualFold(os.Getenv("GITTE_TELEMETRY"), "off") { + return Resolved{} + } + + endpoint := os.Getenv("GITTE_TELEMETRY_URL") + if endpoint == "" && cfg != nil { + endpoint = cfg.Telemetry.Endpoint + } + if endpoint != "" { + headers := map[string]string{} + if cfg != nil { + for k, v := range cfg.Telemetry.Headers { + headers[k] = v + } + } + return Resolved{Enabled: true, Endpoint: endpoint, Headers: headers} + } + + if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") != "" { + return Resolved{Enabled: true, UseSDKEnv: true} + } + + return Resolved{} +} + +// noopErrorHandler swallows OTEL-internal errors (e.g. export failures) so they +// never reach the user or interfere with gitte. +type noopErrorHandler struct{} + +func (noopErrorHandler) Handle(error) {} + +// Init configures the global tracer provider. The returned shutdown function is +// always non-nil and safe to call; it flushes pending spans with a bounded +// timeout. Setup failures degrade to a no-op rather than returning an error. +func Init(ctx context.Context, cfg *config.GitteConfig, version string) (func(), error) { + otel.SetErrorHandler(noopErrorHandler{}) + + r := Resolve(cfg) + if !r.Enabled { + return func() {}, nil + } + + var opts []otlptracehttp.Option + if !r.UseSDKEnv { + opts = append(opts, otlptracehttp.WithEndpointURL(r.Endpoint)) + if len(r.Headers) > 0 { + opts = append(opts, otlptracehttp.WithHeaders(r.Headers)) + } + } + + exporter, err := otlptracehttp.New(ctx, opts...) + if err != nil { + // Never block gitte: disable telemetry on exporter setup failure. + return func() {}, nil + } + + res := resource.NewSchemaless( + attribute.String("service.name", "gitte"), + attribute.String("service.version", version), + attribute.String("os.type", runtime.GOOS), + attribute.String("os.arch", runtime.GOARCH), + ) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + otel.SetTracerProvider(tp) + + return func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), flushTimeout) + defer cancel() + _ = tp.Shutdown(shutdownCtx) + }, nil +} + +// Tracer returns gitte's tracer from the global provider (a no-op tracer when +// telemetry is disabled). +func Tracer() trace.Tracer { + return otel.Tracer(tracerName) +} + +// StartCommandSpan starts the root span for a gitte invocation. +func StartCommandSpan(ctx context.Context, commandPath string, args []string) (context.Context, trace.Span) { + ctx, span := Tracer().Start(ctx, commandPath) + span.SetAttributes( + attribute.String("gitte.command", commandPath), + attribute.StringSlice("gitte.args", args), + ) + return ctx, span +} diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go new file mode 100644 index 00000000..2a87f150 --- /dev/null +++ b/telemetry/telemetry_test.go @@ -0,0 +1,81 @@ +package telemetry + +import ( + "context" + "os" + "testing" + + "github.com/cego/gitte/config" +) + +func TestResolve_Precedence(t *testing.T) { + // Save and clear env that influences resolution. + for _, k := range []string{"GITTE_TELEMETRY", "GITTE_TELEMETRY_URL", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"} { + t.Setenv(k, "") + } + + cfgWith := func(ep string) *config.GitteConfig { + return &config.GitteConfig{Telemetry: config.TelemetryConfig{Endpoint: ep, Headers: map[string]string{"Authorization": "Bearer x"}}} + } + + t.Run("disabled when no endpoint anywhere", func(t *testing.T) { + r := Resolve(&config.GitteConfig{}) + if r.Enabled { + t.Fatal("expected disabled") + } + }) + + t.Run("enabled from config endpoint", func(t *testing.T) { + r := Resolve(cfgWith("https://apm:8200")) + if !r.Enabled || r.Endpoint != "https://apm:8200" || r.UseSDKEnv { + t.Fatalf("got %+v", r) + } + if r.Headers["Authorization"] != "Bearer x" { + t.Fatalf("headers not carried: %+v", r.Headers) + } + }) + + t.Run("GITTE_TELEMETRY_URL overrides config", func(t *testing.T) { + t.Setenv("GITTE_TELEMETRY_URL", "https://override:8200") + r := Resolve(cfgWith("https://apm:8200")) + if r.Endpoint != "https://override:8200" { + t.Fatalf("got %+v", r) + } + }) + + t.Run("GITTE_TELEMETRY=off disables everything", func(t *testing.T) { + t.Setenv("GITTE_TELEMETRY", "off") + t.Setenv("GITTE_TELEMETRY_URL", "https://override:8200") + r := Resolve(cfgWith("https://apm:8200")) + if r.Enabled { + t.Fatalf("expected disabled, got %+v", r) + } + }) + + t.Run("falls back to OTEL env endpoint with UseSDKEnv", func(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel:4318") + r := Resolve(&config.GitteConfig{}) + if !r.Enabled || !r.UseSDKEnv || r.Endpoint != "" { + t.Fatalf("got %+v", r) + } + }) +} + +func TestInit_DisabledReturnsNoopShutdown(t *testing.T) { + t.Setenv("GITTE_TELEMETRY", "off") + shutdown, err := Init(context.Background(), &config.GitteConfig{}, "test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if shutdown == nil { + t.Fatal("shutdown must never be nil") + } + shutdown() // must not panic +} + +func TestStartCommandSpan_NoProviderDoesNotPanic(t *testing.T) { + // With no provider set, Tracer() returns a no-op tracer; span ops are safe. + _, span := StartCommandSpan(context.Background(), "gitte run", []string{"up"}) + span.End() + _ = os.Getenv // keep import +} From c1bde12e24d87888a4680b5609a76298cf333dcf Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 12:07:15 +0200 Subject: [PATCH 03/29] fix(telemetry): add no-op/enabled shutdown tests and clarify header behavior --- telemetry/telemetry.go | 4 ++++ telemetry/telemetry_test.go | 28 ++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 26fa395e..c860bcd6 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -45,6 +45,10 @@ func Resolve(cfg *config.GitteConfig) Resolved { endpoint = cfg.Telemetry.Endpoint } if endpoint != "" { + // Headers always come from config, even when GITTE_TELEMETRY_URL overrides + // the endpoint. This is a known v1 limitation: if you need different headers + // for an override endpoint, use the standard OTEL_EXPORTER_OTLP_* env path + // instead (which lets the SDK read its own config independently). headers := map[string]string{} if cfg != nil { for k, v := range cfg.Telemetry.Headers { diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 2a87f150..1c3190cd 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -2,10 +2,10 @@ package telemetry import ( "context" - "os" "testing" "github.com/cego/gitte/config" + "go.opentelemetry.io/otel" ) func TestResolve_Precedence(t *testing.T) { @@ -41,6 +41,9 @@ func TestResolve_Precedence(t *testing.T) { if r.Endpoint != "https://override:8200" { t.Fatalf("got %+v", r) } + if !r.Enabled { + t.Fatalf("expected Enabled=true when GITTE_TELEMETRY_URL is set, got %+v", r) + } }) t.Run("GITTE_TELEMETRY=off disables everything", func(t *testing.T) { @@ -73,9 +76,30 @@ func TestInit_DisabledReturnsNoopShutdown(t *testing.T) { shutdown() // must not panic } +func TestInit_EnabledReturnsCallableShutdown(t *testing.T) { + // Verify that Init with a valid endpoint returns a non-nil shutdown function + // that can be called without panicking or hanging (bounded 3s flush). + // Note: otlptracehttp.New is lazy — it accepts any URL including unreachable + // endpoints without error, so Init succeeds and returns a real shutdown. + // The exporter-error branch (where New returns an error and Init falls back to + // no-op) cannot be triggered deterministically with the HTTP exporter; the SDK + // silently swallows malformed URLs and connection errors at export time. + t.Setenv("GITTE_TELEMETRY", "") + prev := otel.GetTracerProvider() + t.Cleanup(func() { otel.SetTracerProvider(prev) }) + cfg := &config.GitteConfig{Telemetry: config.TelemetryConfig{Endpoint: "http://localhost:4318"}} + shutdown, err := Init(context.Background(), cfg, "test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if shutdown == nil { + t.Fatal("shutdown must never be nil on the enabled path") + } + shutdown() // must not panic or hang beyond the 3s flush timeout +} + func TestStartCommandSpan_NoProviderDoesNotPanic(t *testing.T) { // With no provider set, Tracer() returns a no-op tracer; span ops are safe. _, span := StartCommandSpan(context.Background(), "gitte run", []string{"up"}) span.End() - _ = os.Getenv // keep import } From 6c979eb8592cdaef9d1f078c21ba28ef96d2aa23 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 12:10:38 +0200 Subject: [PATCH 04/29] feat(cmd): start root telemetry span per invocation --- cmd/root.go | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 6abdd7f0..a23e0329 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,9 +13,12 @@ import ( "github.com/cego/gitte/config" "github.com/cego/gitte/output" "github.com/cego/gitte/state" + "github.com/cego/gitte/telemetry" "charm.land/lipgloss/v2" "github.com/spf13/cobra" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) var ( @@ -32,6 +35,9 @@ var ( globalCwd string globalCtx context.Context globalCancel context.CancelFunc + + globalTelemetryShutdown func() + globalRootSpan trace.Span ) // rootCmd is the base command @@ -54,7 +60,16 @@ with dependency resolution.`, if cmd.Name() == "__complete" || cmd.Name() == "__completeNoDesc" { return nil } - return err + if err != nil { + return err + } + + // Telemetry: best-effort, never blocks. Stores root span context in globalCtx + // so it propagates through the executor into gitops/actions leaf spans. + shutdown, _ := telemetry.Init(globalCtx, globalCfg, cmd.Root().Version) + globalTelemetryShutdown = shutdown + globalCtx, globalRootSpan = telemetry.StartCommandSpan(globalCtx, cmd.CommandPath(), args) + return nil }, } @@ -73,6 +88,7 @@ func Execute() { } }() err := rootCmd.Execute() + finishTelemetry(err) if err != nil { if output.DetectMode(flagNoTTY) == output.ModePlain { fmt.Fprintln(os.Stderr, "error:", err) @@ -83,6 +99,23 @@ func Execute() { } } +// finishTelemetry records the final command status on the root span and flushes +// pending spans. Safe to call when telemetry is disabled (handles are nil). +func finishTelemetry(err error) { + if globalRootSpan != nil { + if err != nil { + globalRootSpan.RecordError(err) + globalRootSpan.SetStatus(codes.Error, err.Error()) + } else { + globalRootSpan.SetStatus(codes.Ok, "") + } + globalRootSpan.End() + } + if globalTelemetryShutdown != nil { + globalTelemetryShutdown() + } +} + func init() { rootCmd.PersistentFlags().StringVar(&flagConfigPath, "config", "", "path to .gitte.yml (default: auto-discover)") rootCmd.PersistentFlags().BoolVar(&flagNoTTY, "no-tty", false, "disable TUI (plain output)") From 9af2dd8eea77934057a9fcb61031775b39c744fb Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 12:15:00 +0200 Subject: [PATCH 05/29] feat(gitops): add per-project sync spans with git context --- gitops/gitops.go | 44 ++++++++++++++++++++++++++++++++++++---- gitops/telemetry_test.go | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 gitops/telemetry_test.go diff --git a/gitops/gitops.go b/gitops/gitops.go index 20cb5959..e3ac6668 100644 --- a/gitops/gitops.go +++ b/gitops/gitops.go @@ -18,6 +18,11 @@ import ( "github.com/cego/gitte/config" "github.com/cego/gitte/executor" "github.com/cego/gitte/output" + "github.com/cego/gitte/telemetry" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) // parallelLimit returns the effective parallelization cap for gitops clone/pull @@ -182,10 +187,20 @@ func syncProject( setDetail func(string), addPrompt func(CheckoutPrompt), warnFn func(string), -) error { - localDir, err := config.LocalDirForRemote(proj.Remote) - if err != nil { - return err +) (err error) { + ctx, span := telemetry.Tracer().Start(ctx, "gitops.sync") + span.SetAttributes(attribute.String("gitte.repo", name)) + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + }() + + localDir, lerr := config.LocalDirForRemote(proj.Remote) + if lerr != nil { + return lerr } projectPath := filepath.Join(cwd, localDir) @@ -245,6 +260,7 @@ func syncProject( if err != nil { return err } + setGitContextAttrs(span, name, currentBranch, getHeadSHA(ctx, projectPath), dirty) if dirty { setDetail("skipped") if currentBranch != defaultBranch { @@ -407,6 +423,26 @@ func staleDays(ctx context.Context, dir, defaultBranch string) int { return 0 } +// setGitContextAttrs records non-PII git context on a span. repo is the repo +// name/path (never the full remote URL). +func setGitContextAttrs(span trace.Span, repo, branch, sha string, dirty bool) { + span.SetAttributes( + attribute.String("gitte.repo", repo), + attribute.String("git.branch", branch), + attribute.String("git.sha", sha), + attribute.Bool("git.dirty", dirty), + ) +} + +// getHeadSHA returns the short HEAD commit SHA, or "" if it cannot be determined. +func getHeadSHA(ctx context.Context, dir string) string { + res, err := executor.ExecuteSyncInDir(ctx, dir, "git", "rev-parse", "--short", "HEAD") + if err != nil || res.ExitCode != 0 { + return "" + } + return strings.TrimSpace(string(res.Stdout)) +} + // ── git helpers ────────────────────────────────────────────────────────────── const fetchTimeout = 60 * time.Second diff --git a/gitops/telemetry_test.go b/gitops/telemetry_test.go new file mode 100644 index 00000000..f80aec33 --- /dev/null +++ b/gitops/telemetry_test.go @@ -0,0 +1,43 @@ +package gitops + +import ( + "context" + "testing" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel" +) + +func TestSetGitContextAttrs(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + otel.SetTracerProvider(tp) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + + _, span := tp.Tracer("test").Start(context.Background(), "gitops.sync") + setGitContextAttrs(span, "group/repo", "main", "abc123", true) + span.End() + + spans := exp.GetSpans() + if len(spans) != 1 { + t.Fatalf("got %d spans, want 1", len(spans)) + } + attrs := map[string]string{} + dirty := false + for _, kv := range spans[0].Attributes { + switch kv.Key { + case "gitte.repo": + attrs["repo"] = kv.Value.AsString() + case "git.branch": + attrs["branch"] = kv.Value.AsString() + case "git.sha": + attrs["sha"] = kv.Value.AsString() + case "git.dirty": + dirty = kv.Value.AsBool() + } + } + if attrs["repo"] != "group/repo" || attrs["branch"] != "main" || attrs["sha"] != "abc123" || !dirty { + t.Fatalf("attrs = %+v dirty=%v", attrs, dirty) + } +} From 4c2765d0142d36133239bac7f802b16849dfc976 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 12:18:36 +0200 Subject: [PATCH 06/29] feat(actions): add per-task action spans with exit code --- actions/runner.go | 28 +++++++++++++++++++++++++++- actions/telemetry_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 actions/telemetry_test.go diff --git a/actions/runner.go b/actions/runner.go index 2c4a8e83..70f08864 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -19,6 +19,11 @@ import ( "github.com/cego/gitte/features" "github.com/cego/gitte/output" "github.com/cego/gitte/state" + "github.com/cego/gitte/telemetry" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) // RunActions executes planned action tasks. @@ -250,6 +255,15 @@ func taskName(key GroupKey) string { return fmt.Sprintf("%s:%s:%s", key.Project, key.Action, key.Group) } +// setActionAttrs records non-PII action context on a span. +func setActionAttrs(span trace.Span, taskName, project, command string) { + span.SetAttributes( + attribute.String("gitte.task", taskName), + attribute.String("gitte.project", project), + attribute.String("gitte.command", command), + ) +} + func runGroupTask( ctx context.Context, cfg *config.GitteConfig, @@ -261,7 +275,17 @@ func runGroupTask( cmds []string, searchFors []config.SearchFor, handler executor.OutputHandler, -) error { +) (err error) { + ctx, span := telemetry.Tracer().Start(ctx, "action.run") + setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + }() + if len(cmds) == 0 { return fmt.Errorf("empty command for task %s", taskName) } @@ -291,6 +315,8 @@ func runGroupTask( return err } + span.SetAttributes(attribute.Int("gitte.exit_code", res.ExitCode)) + if res.ExitCode != 0 { return fmt.Errorf("command exited with code %d", res.ExitCode) } diff --git a/actions/telemetry_test.go b/actions/telemetry_test.go new file mode 100644 index 00000000..f5e50ca8 --- /dev/null +++ b/actions/telemetry_test.go @@ -0,0 +1,33 @@ +package actions + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestSetActionAttrs(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + otel.SetTracerProvider(tp) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + + _, span := tp.Tracer("test").Start(context.Background(), "action.run") + setActionAttrs(span, "proj:up:default", "proj", "docker compose up") + span.End() + + spans := exp.GetSpans() + if len(spans) != 1 { + t.Fatalf("got %d spans, want 1", len(spans)) + } + got := map[string]string{} + for _, kv := range spans[0].Attributes { + got[string(kv.Key)] = kv.Value.AsString() + } + if got["gitte.task"] != "proj:up:default" || got["gitte.project"] != "proj" || got["gitte.command"] != "docker compose up" { + t.Fatalf("attrs = %+v", got) + } +} From 726f7df1d4f709358e1cb29f0a0c6b9c212767c5 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 13:28:23 +0200 Subject: [PATCH 07/29] docs: document telemetry configuration --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index f773fbb1..04986f8b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Gitte keeps all your repos in sync, runs startup checks to verify your local mac - [Commands](#commands) - [Configuration](#configuration) - [Environment variables](#environment-variables) +- [Telemetry](#telemetry) - [Global flags](#global-flags) - [State and override files](#state-and-override-files) @@ -170,6 +171,36 @@ See [docs/config.md](./docs/config.md) for the full configuration reference. --- +## Telemetry + +Gitte can export OpenTelemetry traces to an OTLP/HTTP endpoint (e.g. Elastic +APM) to help debug failures. Traces capture the command run, per-repo git +context (branch, commit SHA, dirty state), and per-task outcomes with errors. +No PII is collected (no hostname, OS username, or full remote URLs). + +Enable it via the shared config: + +```yaml +telemetry: + endpoint: https://apm.example.com:8200 + headers: + Authorization: "Bearer " # or: "ApiKey " +``` + +Environment variables: + +| Variable | Effect | +|---|---| +| `GITTE_TELEMETRY=off` | Disable telemetry locally (kill-switch) | +| `GITTE_TELEMETRY_URL` | Override the endpoint | +| `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` | Standard OTEL env vars, honored as a fallback when no gitte endpoint is set | + +Precedence: `GITTE_TELEMETRY=off` > `GITTE_TELEMETRY_URL` > config endpoint > +`OTEL_EXPORTER_OTLP_*`. Telemetry is best-effort and never blocks or slows +gitte; export failures are silently ignored. + +--- + ## Global flags ``` From b9fdd245df831a42e8c755134322160e07311cfa Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 13:38:43 +0200 Subject: [PATCH 08/29] style: fix import ordering in gitops telemetry test --- gitops/telemetry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitops/telemetry_test.go b/gitops/telemetry_test.go index f80aec33..bb6ffd46 100644 --- a/gitops/telemetry_test.go +++ b/gitops/telemetry_test.go @@ -4,9 +4,9 @@ import ( "context" "testing" + "go.opentelemetry.io/otel" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" - "go.opentelemetry.io/otel" ) func TestSetGitContextAttrs(t *testing.T) { From c047579fe145a473cb31a9d5f6ed55b5f2662fba Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 13:43:15 +0200 Subject: [PATCH 09/29] feat(telemetry): attach OS username and hostname to traces --- README.md | 4 +++- actions/runner.go | 2 +- gitops/gitops.go | 4 ++-- telemetry/telemetry.go | 33 ++++++++++++++++++++++++++------ telemetry/telemetry_test.go | 38 +++++++++++++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 04986f8b..949463d9 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,9 @@ See [docs/config.md](./docs/config.md) for the full configuration reference. Gitte can export OpenTelemetry traces to an OTLP/HTTP endpoint (e.g. Elastic APM) to help debug failures. Traces capture the command run, per-repo git context (branch, commit SHA, dirty state), and per-task outcomes with errors. -No PII is collected (no hostname, OS username, or full remote URLs). +To identify which developer and machine hit a failure, the OS username +(`user.name`) and hostname (`host.name`) are attached to every trace. Full +remote URLs and command environment values are never collected. Enable it via the shared config: diff --git a/actions/runner.go b/actions/runner.go index 70f08864..7fc4f1d3 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -255,7 +255,7 @@ func taskName(key GroupKey) string { return fmt.Sprintf("%s:%s:%s", key.Project, key.Action, key.Group) } -// setActionAttrs records non-PII action context on a span. +// setActionAttrs records action context on a span. func setActionAttrs(span trace.Span, taskName, project, command string) { span.SetAttributes( attribute.String("gitte.task", taskName), diff --git a/gitops/gitops.go b/gitops/gitops.go index e3ac6668..6a02ed9f 100644 --- a/gitops/gitops.go +++ b/gitops/gitops.go @@ -423,8 +423,8 @@ func staleDays(ctx context.Context, dir, defaultBranch string) int { return 0 } -// setGitContextAttrs records non-PII git context on a span. repo is the repo -// name/path (never the full remote URL). +// setGitContextAttrs records git context on a span. repo is the repo name/path +// (never the full remote URL, which can embed credentials). func setGitContextAttrs(span trace.Span, repo, branch, sha string, dirty bool) { span.SetAttributes( attribute.String("gitte.repo", repo), diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index c860bcd6..4c4c85da 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -7,6 +7,7 @@ package telemetry import ( "context" "os" + "os/user" "runtime" "strings" "time" @@ -71,6 +72,26 @@ type noopErrorHandler struct{} func (noopErrorHandler) Handle(error) {} +// resourceAttributes builds the resource attributes attached to every span. +// username and hostname identify which developer and machine produced the +// trace (the primary signal for debugging machine-specific failures); both are +// best-effort and omitted when they cannot be resolved. +func resourceAttributes(version, username, hostname string) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("service.name", "gitte"), + attribute.String("service.version", version), + attribute.String("os.type", runtime.GOOS), + attribute.String("os.arch", runtime.GOARCH), + } + if username != "" { + attrs = append(attrs, attribute.String("user.name", username)) + } + if hostname != "" { + attrs = append(attrs, attribute.String("host.name", hostname)) + } + return attrs +} + // Init configures the global tracer provider. The returned shutdown function is // always non-nil and safe to call; it flushes pending spans with a bounded // timeout. Setup failures degrade to a no-op rather than returning an error. @@ -96,12 +117,12 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) (func(), return func() {}, nil } - res := resource.NewSchemaless( - attribute.String("service.name", "gitte"), - attribute.String("service.version", version), - attribute.String("os.type", runtime.GOOS), - attribute.String("os.arch", runtime.GOARCH), - ) + username := "" + if u, uerr := user.Current(); uerr == nil { + username = u.Username + } + hostname, _ := os.Hostname() + res := resource.NewSchemaless(resourceAttributes(version, username, hostname)...) tp := sdktrace.NewTracerProvider( sdktrace.WithBatcher(exporter), diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 1c3190cd..b6f2352c 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -6,8 +6,46 @@ import ( "github.com/cego/gitte/config" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" ) +func TestResourceAttributes(t *testing.T) { + find := func(attrs []attribute.KeyValue, key string) (string, bool) { + for _, a := range attrs { + if string(a.Key) == key { + return a.Value.AsString(), true + } + } + return "", false + } + + t.Run("includes username and hostname when resolved", func(t *testing.T) { + attrs := resourceAttributes("1.2.3", "alice", "dev-box") + if v, _ := find(attrs, "service.name"); v != "gitte" { + t.Errorf("service.name = %q, want gitte", v) + } + if v, _ := find(attrs, "service.version"); v != "1.2.3" { + t.Errorf("service.version = %q, want 1.2.3", v) + } + if v, ok := find(attrs, "user.name"); !ok || v != "alice" { + t.Errorf("user.name = %q (present=%v), want alice", v, ok) + } + if v, ok := find(attrs, "host.name"); !ok || v != "dev-box" { + t.Errorf("host.name = %q (present=%v), want dev-box", v, ok) + } + }) + + t.Run("omits username and hostname when empty", func(t *testing.T) { + attrs := resourceAttributes("1.2.3", "", "") + if _, ok := find(attrs, "user.name"); ok { + t.Error("user.name should be omitted when empty") + } + if _, ok := find(attrs, "host.name"); ok { + t.Error("host.name should be omitted when empty") + } + }) +} + func TestResolve_Precedence(t *testing.T) { // Save and clear env that influences resolution. for _, k := range []string{"GITTE_TELEMETRY", "GITTE_TELEMETRY_URL", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"} { From c7f80440ed33249cd21c1af136cca7764f2c2059 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 14:33:36 +0200 Subject: [PATCH 10/29] fix(telemetry): skip git exec when disabled, shorten flush, clarify recorded data - Guard getHeadSHA behind span.IsRecording() so it never execs when telemetry is off (was running git rev-parse on every repo every sync regardless) - Shorten flush timeout 3s -> 1s so an unreachable endpoint adds at most 1s on exit - Document that CLI args and action command lines are recorded; advise keeping secrets in env (never exported) - Drop redundant gitte.repo double-set; drop always-nil error return from Init --- README.md | 15 ++++++++++----- cmd/root.go | 3 +-- gitops/gitops.go | 14 +++++++++----- gitops/telemetry_test.go | 6 ++---- telemetry/telemetry.go | 20 ++++++++++++-------- telemetry/telemetry_test.go | 14 ++++---------- 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 949463d9..74912e9d 100644 --- a/README.md +++ b/README.md @@ -174,11 +174,16 @@ See [docs/config.md](./docs/config.md) for the full configuration reference. ## Telemetry Gitte can export OpenTelemetry traces to an OTLP/HTTP endpoint (e.g. Elastic -APM) to help debug failures. Traces capture the command run, per-repo git -context (branch, commit SHA, dirty state), and per-task outcomes with errors. -To identify which developer and machine hit a failure, the OS username -(`user.name`) and hostname (`host.name`) are attached to every trace. Full -remote URLs and command environment values are never collected. +APM) to help debug failures. Traces capture per-repo git context (branch, +commit SHA, dirty state) and per-task outcomes with errors. To identify which +developer and machine hit a failure, the OS username (`user.name`) and hostname +(`host.name`) are attached to every trace. + +**What is recorded:** the gitte CLI arguments and each action's command line are +exported as span attributes (this is intentional — knowing what ran is the +point). Keep secrets out of action command definitions and CLI arguments; pass +them through environment variables, which are **not** exported. Full remote +URLs are never collected either (repos are identified by name only). Enable it via the shared config: diff --git a/cmd/root.go b/cmd/root.go index a23e0329..c5a3be95 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -66,8 +66,7 @@ with dependency resolution.`, // Telemetry: best-effort, never blocks. Stores root span context in globalCtx // so it propagates through the executor into gitops/actions leaf spans. - shutdown, _ := telemetry.Init(globalCtx, globalCfg, cmd.Root().Version) - globalTelemetryShutdown = shutdown + globalTelemetryShutdown = telemetry.Init(globalCtx, globalCfg, cmd.Root().Version) globalCtx, globalRootSpan = telemetry.StartCommandSpan(globalCtx, cmd.CommandPath(), args) return nil }, diff --git a/gitops/gitops.go b/gitops/gitops.go index 6a02ed9f..bf5272aa 100644 --- a/gitops/gitops.go +++ b/gitops/gitops.go @@ -260,7 +260,11 @@ func syncProject( if err != nil { return err } - setGitContextAttrs(span, name, currentBranch, getHeadSHA(ctx, projectPath), dirty) + // Guard on IsRecording so getHeadSHA (a git exec) never runs when telemetry + // is disabled — the span is non-recording and would discard the attributes. + if span.IsRecording() { + setGitContextAttrs(span, currentBranch, getHeadSHA(ctx, projectPath), dirty) + } if dirty { setDetail("skipped") if currentBranch != defaultBranch { @@ -423,11 +427,11 @@ func staleDays(ctx context.Context, dir, defaultBranch string) int { return 0 } -// setGitContextAttrs records git context on a span. repo is the repo name/path -// (never the full remote URL, which can embed credentials). -func setGitContextAttrs(span trace.Span, repo, branch, sha string, dirty bool) { +// setGitContextAttrs records git context on a span. The caller sets gitte.repo +// separately (the repo name/path, never the full remote URL which can embed +// credentials) so it is present on every span, including early-return paths. +func setGitContextAttrs(span trace.Span, branch, sha string, dirty bool) { span.SetAttributes( - attribute.String("gitte.repo", repo), attribute.String("git.branch", branch), attribute.String("git.sha", sha), attribute.Bool("git.dirty", dirty), diff --git a/gitops/telemetry_test.go b/gitops/telemetry_test.go index bb6ffd46..665874ef 100644 --- a/gitops/telemetry_test.go +++ b/gitops/telemetry_test.go @@ -16,7 +16,7 @@ func TestSetGitContextAttrs(t *testing.T) { t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) _, span := tp.Tracer("test").Start(context.Background(), "gitops.sync") - setGitContextAttrs(span, "group/repo", "main", "abc123", true) + setGitContextAttrs(span, "main", "abc123", true) span.End() spans := exp.GetSpans() @@ -27,8 +27,6 @@ func TestSetGitContextAttrs(t *testing.T) { dirty := false for _, kv := range spans[0].Attributes { switch kv.Key { - case "gitte.repo": - attrs["repo"] = kv.Value.AsString() case "git.branch": attrs["branch"] = kv.Value.AsString() case "git.sha": @@ -37,7 +35,7 @@ func TestSetGitContextAttrs(t *testing.T) { dirty = kv.Value.AsBool() } } - if attrs["repo"] != "group/repo" || attrs["branch"] != "main" || attrs["sha"] != "abc123" || !dirty { + if attrs["branch"] != "main" || attrs["sha"] != "abc123" || !dirty { t.Fatalf("attrs = %+v dirty=%v", attrs, dirty) } } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 4c4c85da..b84094fa 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -24,7 +24,10 @@ import ( const tracerName = "github.com/cego/gitte" -const flushTimeout = 3 * time.Second +// flushTimeout bounds how long exit can block flushing spans. Kept short so an +// enabled-but-unreachable endpoint (e.g. laptop with the VPN off) adds at most +// this delay to every command. +const flushTimeout = 1 * time.Second // Resolved is the outcome of resolving telemetry settings from config + env. type Resolved struct { @@ -92,15 +95,16 @@ func resourceAttributes(version, username, hostname string) []attribute.KeyValue return attrs } -// Init configures the global tracer provider. The returned shutdown function is -// always non-nil and safe to call; it flushes pending spans with a bounded -// timeout. Setup failures degrade to a no-op rather than returning an error. -func Init(ctx context.Context, cfg *config.GitteConfig, version string) (func(), error) { +// Init configures the global tracer provider and returns a shutdown function +// that flushes pending spans with a bounded timeout. The returned function is +// always non-nil and safe to call; setup failures and disabled telemetry both +// degrade to a no-op shutdown. +func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { otel.SetErrorHandler(noopErrorHandler{}) r := Resolve(cfg) if !r.Enabled { - return func() {}, nil + return func() {} } var opts []otlptracehttp.Option @@ -114,7 +118,7 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) (func(), exporter, err := otlptracehttp.New(ctx, opts...) if err != nil { // Never block gitte: disable telemetry on exporter setup failure. - return func() {}, nil + return func() {} } username := "" @@ -135,7 +139,7 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) (func(), shutdownCtx, cancel := context.WithTimeout(context.Background(), flushTimeout) defer cancel() _ = tp.Shutdown(shutdownCtx) - }, nil + } } // Tracer returns gitte's tracer from the global provider (a no-op tracer when diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index b6f2352c..b239916c 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -104,10 +104,7 @@ func TestResolve_Precedence(t *testing.T) { func TestInit_DisabledReturnsNoopShutdown(t *testing.T) { t.Setenv("GITTE_TELEMETRY", "off") - shutdown, err := Init(context.Background(), &config.GitteConfig{}, "test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + shutdown := Init(context.Background(), &config.GitteConfig{}, "test") if shutdown == nil { t.Fatal("shutdown must never be nil") } @@ -116,7 +113,7 @@ func TestInit_DisabledReturnsNoopShutdown(t *testing.T) { func TestInit_EnabledReturnsCallableShutdown(t *testing.T) { // Verify that Init with a valid endpoint returns a non-nil shutdown function - // that can be called without panicking or hanging (bounded 3s flush). + // that can be called without panicking or hanging (bounded flush timeout). // Note: otlptracehttp.New is lazy — it accepts any URL including unreachable // endpoints without error, so Init succeeds and returns a real shutdown. // The exporter-error branch (where New returns an error and Init falls back to @@ -126,14 +123,11 @@ func TestInit_EnabledReturnsCallableShutdown(t *testing.T) { prev := otel.GetTracerProvider() t.Cleanup(func() { otel.SetTracerProvider(prev) }) cfg := &config.GitteConfig{Telemetry: config.TelemetryConfig{Endpoint: "http://localhost:4318"}} - shutdown, err := Init(context.Background(), cfg, "test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + shutdown := Init(context.Background(), cfg, "test") if shutdown == nil { t.Fatal("shutdown must never be nil on the enabled path") } - shutdown() // must not panic or hang beyond the 3s flush timeout + shutdown() // must not panic or hang beyond the flush timeout } func TestStartCommandSpan_NoProviderDoesNotPanic(t *testing.T) { From ba559cf5e5322bea2a2daec085238f4ba48137dc Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Thu, 25 Jun 2026 14:53:32 +0200 Subject: [PATCH 11/29] fix(telemetry): address review comments - Set the global OTEL error handler only when telemetry is enabled - Correct finishTelemetry doc comment (handles are nil only when uninitialized) - Restore the previous tracer provider in gitops/actions attribute tests - Document the telemetry config block in docs/config.md --- actions/telemetry_test.go | 6 +++++- cmd/root.go | 3 ++- docs/config.md | 34 ++++++++++++++++++++++++++++++++++ gitops/telemetry_test.go | 6 +++++- telemetry/telemetry.go | 5 +++-- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/actions/telemetry_test.go b/actions/telemetry_test.go index f5e50ca8..b38b40fb 100644 --- a/actions/telemetry_test.go +++ b/actions/telemetry_test.go @@ -12,8 +12,12 @@ import ( func TestSetActionAttrs(t *testing.T) { exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + prev := otel.GetTracerProvider() otel.SetTracerProvider(tp) - t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + t.Cleanup(func() { + otel.SetTracerProvider(prev) + _ = tp.Shutdown(context.Background()) + }) _, span := tp.Tracer("test").Start(context.Background(), "action.run") setActionAttrs(span, "proj:up:default", "proj", "docker compose up") diff --git a/cmd/root.go b/cmd/root.go index c5a3be95..108cbce1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -99,7 +99,8 @@ func Execute() { } // finishTelemetry records the final command status on the root span and flushes -// pending spans. Safe to call when telemetry is disabled (handles are nil). +// pending spans. Safe to call when telemetry was never initialized (e.g. +// completion commands or an early config failure), where the handles remain nil. func finishTelemetry(err error) { if globalRootSpan != nil { if err != nil { diff --git a/docs/config.md b/docs/config.md index e3e29268..5b888d63 100644 --- a/docs/config.md +++ b/docs/config.md @@ -18,6 +18,7 @@ An optional `.gitte-override.yml` in the same directory is deep-merged on top, u - [searchFor](#searchfor) - [feature\_gates](#feature_gates) - [sources](#sources) +- [telemetry](#telemetry) - [Remote configuration](#remote-configuration) --- @@ -34,6 +35,7 @@ sources: # auto-discovery sources (optional) searchFor: # global output pattern matching (optional) actionOverride: # per-action overrides (optional) retry: # global retry defaults (optional) +telemetry: # OpenTelemetry trace export (optional) ``` --- @@ -406,6 +408,38 @@ gitte run up --discover # discover, then sync, then run actions --- +## telemetry + +Gitte can export OpenTelemetry traces over OTLP/HTTP to an OTLP-compatible backend (e.g. Elastic APM) to help debug failures. Telemetry is enabled whenever an endpoint is resolved. + +```yaml +telemetry: + endpoint: https://apm.example.com:8200 # OTLP/HTTP endpoint + headers: # arbitrary export headers (optional) + Authorization: "Bearer " # or: "ApiKey " +``` + +| Field | Description | +|-------|-------------| +| `endpoint` | OTLP/HTTP endpoint to export spans to. Telemetry is enabled when this resolves to a non-empty value. | +| `headers` | Map of HTTP headers attached to every export request — typically authentication (`Authorization`). | + +Each invocation produces one trace: a root span for the command, child spans for each repo sync (branch, commit SHA, dirty flag) and each action task (command, exit code), with errors recorded on the relevant span. The OS username (`user.name`) and hostname (`host.name`) are attached to every trace to identify which developer and machine produced it. + +The gitte CLI arguments and each action's command line are exported as span attributes. Keep secrets out of action command definitions and CLI arguments — pass them via environment variables, which are not exported. Full remote URLs are never collected (repos are identified by name only). + +Environment variables override or disable telemetry: + +| Variable | Effect | +|----------|--------| +| `GITTE_TELEMETRY=off` | Disable telemetry locally (kill-switch) | +| `GITTE_TELEMETRY_URL` | Override the endpoint | +| `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` | Standard OTEL env vars, honored as a fallback when no gitte endpoint is set | + +Precedence: `GITTE_TELEMETRY=off` > `GITTE_TELEMETRY_URL` > config `endpoint` > `OTEL_EXPORTER_OTLP_*`. Telemetry is best-effort and never blocks or slows gitte; export failures are silently ignored and flushing on exit is time-bounded. + +--- + ## Remote configuration Gitte can load its configuration from a remote git repository. Create a `.gitte-env` file alongside `.gitte.yml`: diff --git a/gitops/telemetry_test.go b/gitops/telemetry_test.go index 665874ef..0269fc9f 100644 --- a/gitops/telemetry_test.go +++ b/gitops/telemetry_test.go @@ -12,8 +12,12 @@ import ( func TestSetGitContextAttrs(t *testing.T) { exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + prev := otel.GetTracerProvider() otel.SetTracerProvider(tp) - t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + t.Cleanup(func() { + otel.SetTracerProvider(prev) + _ = tp.Shutdown(context.Background()) + }) _, span := tp.Tracer("test").Start(context.Background(), "gitops.sync") setGitContextAttrs(span, "main", "abc123", true) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index b84094fa..0930d3ed 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -100,13 +100,14 @@ func resourceAttributes(version, username, hostname string) []attribute.KeyValue // always non-nil and safe to call; setup failures and disabled telemetry both // degrade to a no-op shutdown. func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { - otel.SetErrorHandler(noopErrorHandler{}) - r := Resolve(cfg) if !r.Enabled { return func() {} } + // Only mutate process-wide OTEL state once telemetry is known to be enabled. + otel.SetErrorHandler(noopErrorHandler{}) + var opts []otlptracehttp.Option if !r.UseSDKEnv { opts = append(opts, otlptracehttp.WithEndpointURL(r.Endpoint)) From 5950e1b55b1985caf45c04969a05f457a444f42f Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 11:50:53 +0200 Subject: [PATCH 12/29] feat(telemetry): add GITTE_TELEMETRY_DEBUG to surface export errors --- telemetry/telemetry.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 0930d3ed..4e6cee3e 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -6,6 +6,7 @@ package telemetry import ( "context" + "fmt" "os" "os/user" "runtime" @@ -75,6 +76,16 @@ type noopErrorHandler struct{} func (noopErrorHandler) Handle(error) {} +// debugErrorHandler logs OTEL-internal errors to stderr. Enabled via +// GITTE_TELEMETRY_DEBUG so export failures (auth, redirects, connectivity) are +// visible when diagnosing why traces aren't arriving — otherwise they are +// silently swallowed. +type debugErrorHandler struct{} + +func (debugErrorHandler) Handle(err error) { + fmt.Fprintf(os.Stderr, "[telemetry] %v\n", err) +} + // resourceAttributes builds the resource attributes attached to every span. // username and hostname identify which developer and machine produced the // trace (the primary signal for debugging machine-specific failures); both are @@ -106,7 +117,12 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { } // Only mutate process-wide OTEL state once telemetry is known to be enabled. - otel.SetErrorHandler(noopErrorHandler{}) + // GITTE_TELEMETRY_DEBUG surfaces export errors to stderr for diagnostics. + if os.Getenv("GITTE_TELEMETRY_DEBUG") != "" { + otel.SetErrorHandler(debugErrorHandler{}) + } else { + otel.SetErrorHandler(noopErrorHandler{}) + } var opts []otlptracehttp.Option if !r.UseSDKEnv { From e33c1df61a8dec1767a5af013766296dad62a0dc Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 11:55:05 +0200 Subject: [PATCH 13/29] feat(telemetry): add OTEL logs provider and correlated log output handler --- go.mod | 3 ++ go.sum | 8 ++++ telemetry/logs.go | 91 ++++++++++++++++++++++++++++++++++++++++++ telemetry/logs_test.go | 50 +++++++++++++++++++++++ telemetry/telemetry.go | 30 ++++++++++++++ 5 files changed, 182 insertions(+) create mode 100644 telemetry/logs.go create mode 100644 telemetry/logs_test.go diff --git a/go.mod b/go.mod index 6a6d2441..73ceab5f 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,11 @@ require ( github.com/spf13/cobra v1.10.2 github.com/zalando/go-keyring v0.2.8 go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 + go.opentelemetry.io/otel/log v0.20.0 go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/sdk/log v0.20.0 go.opentelemetry.io/otel/trace v1.44.0 golang.org/x/term v0.43.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index f8c8172f..bcedc2f8 100644 --- a/go.sum +++ b/go.sum @@ -89,14 +89,22 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0 h1:owlhcJ3QO3X0YTDTCcDZ4V+6aVDkWbNmBoQ5NUp7Oww= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0/go.mod h1:MP4eemTiI9zC8fgg+DYynhYDYf3ba72S376TvP+Ye0Q= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/otel/log v0.20.0 h1:/5i0vuHxCLWUfChWG41K9wkM0jafruPw9NU1/RCJirs= +go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/log v0.20.0 h1:vM3xI7TQgKPiSghe6urZtAkyFY7SodrSpC83CffDFuY= +go.opentelemetry.io/otel/sdk/log v0.20.0/go.mod h1:Knej2nmsTUzN79T2eeXdRsjjPcoxoq2pUyUHz9TFyyU= +go.opentelemetry.io/otel/sdk/log/logtest v0.20.0 h1:OqdRZ1guyzamK3M6LlRsmGqRrjkHWw6WZOKKli5ELpg= +go.opentelemetry.io/otel/sdk/log/logtest v0.20.0/go.mod h1:PuMIlm7zAt7c3z8zfOI5ox4iT1Z87We+PF6YoINux/M= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= diff --git a/telemetry/logs.go b/telemetry/logs.go new file mode 100644 index 00000000..78d9abf7 --- /dev/null +++ b/telemetry/logs.go @@ -0,0 +1,91 @@ +package telemetry + +import ( + "context" + "sync" + + "github.com/cego/gitte/executor" + + "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/trace" +) + +// SpanRegistry maps a task name to the SpanContext of the span representing that +// task, so output lines (which arrive on a separate goroutine without the task +// span in their context) can be correlated to the right span. +type SpanRegistry struct { + mu sync.RWMutex + m map[string]trace.SpanContext +} + +// NewSpanRegistry returns an empty, concurrency-safe registry. +func NewSpanRegistry() *SpanRegistry { + return &SpanRegistry{m: make(map[string]trace.SpanContext)} +} + +func (r *SpanRegistry) Set(task string, sc trace.SpanContext) { + r.mu.Lock() + r.m[task] = sc + r.mu.Unlock() +} + +func (r *SpanRegistry) Get(task string) (trace.SpanContext, bool) { + r.mu.RLock() + sc, ok := r.m[task] + r.mu.RUnlock() + return sc, ok +} + +func (r *SpanRegistry) Delete(task string) { + r.mu.Lock() + delete(r.m, task) + r.mu.Unlock() +} + +// logHandler forwards output to inner and emits a correlated OTEL log record. +type logHandler struct { + inner executor.OutputHandler + reg *SpanRegistry + lgr log.Logger +} + +// LogOutputHandler wraps inner so that every output line is also emitted as an +// OTEL log record correlated (via reg) to the span for output.CmdName. When +// logs are disabled the global logger provider is a no-op, so this is safe and +// cheap; output is always forwarded to inner unchanged. +func LogOutputHandler(inner executor.OutputHandler, reg *SpanRegistry) executor.OutputHandler { + return &logHandler{ + inner: inner, + reg: reg, + lgr: global.GetLoggerProvider().Logger("github.com/cego/gitte"), + } +} + +func (h *logHandler) HandleOutput(ctx context.Context, out executor.Output) error { + h.emit(ctx, out) + return h.inner.HandleOutput(ctx, out) +} + +func (h *logHandler) emit(ctx context.Context, out executor.Output) { + var rec log.Record + rec.SetBody(log.StringValue(string(out.Output))) + sev := log.SeverityInfo + if out.Stream == executor.StderrStream { + sev = log.SeverityWarn + } + rec.SetSeverity(sev) + rec.AddAttributes( + log.String("gitte.task", out.CmdName), + log.String("stream", string(out.Stream)), + ) + if len(out.Output) >= 6 && string(out.Output[:6]) == "[HINT]" { + rec.AddAttributes(log.Bool("gitte.hint", true)) + } + // Correlate to the task span if we know it. + emitCtx := ctx + if sc, ok := h.reg.Get(out.CmdName); ok && sc.IsValid() { + emitCtx = trace.ContextWithSpanContext(ctx, sc) + } + h.lgr.Emit(emitCtx, rec) +} diff --git a/telemetry/logs_test.go b/telemetry/logs_test.go new file mode 100644 index 00000000..61849430 --- /dev/null +++ b/telemetry/logs_test.go @@ -0,0 +1,50 @@ +package telemetry + +import ( + "context" + "testing" + + "github.com/cego/gitte/executor" + "go.opentelemetry.io/otel/trace" +) + +func TestSpanRegistry_SetGetDelete(t *testing.T) { + reg := NewSpanRegistry() + sc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: trace.TraceID{1}, + SpanID: trace.SpanID{2}, + }) + reg.Set("proj:build:sn", sc) + got, ok := reg.Get("proj:build:sn") + if !ok || got.TraceID() != sc.TraceID() { + t.Fatalf("Get = %v, %v; want %v", got, ok, sc) + } + reg.Delete("proj:build:sn") + if _, ok := reg.Get("proj:build:sn"); ok { + t.Fatal("expected entry removed after Delete") + } +} + +type recordingHandler struct{ lines []string } + +func (r *recordingHandler) HandleOutput(_ context.Context, o executor.Output) error { + r.lines = append(r.lines, string(o.Output)) + return nil +} + +func TestLogOutputHandler_ForwardsUnchanged(t *testing.T) { + // With logs disabled (no provider), the wrapper must still forward output + // to the inner handler and never error. + inner := &recordingHandler{} + reg := NewSpanRegistry() + h := LogOutputHandler(inner, reg) + err := h.HandleOutput(context.Background(), executor.Output{ + Output: []byte("hello"), CmdName: "proj:build:sn", Stream: executor.StdoutStream, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(inner.lines) != 1 || inner.lines[0] != "hello" { + t.Fatalf("inner did not receive line: %+v", inner.lines) + } +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 4e6cee3e..acd5efaa 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -17,8 +17,11 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + otlplog "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + otellog "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/sdk/resource" + sdklog "go.opentelemetry.io/otel/sdk/log" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" ) @@ -152,13 +155,40 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { ) otel.SetTracerProvider(tp) + var lp *sdklog.LoggerProvider + if logsEnabled() { + var logOpts []otlplog.Option + if !r.UseSDKEnv { + logOpts = append(logOpts, otlplog.WithEndpointURL(r.Endpoint)) + if len(r.Headers) > 0 { + logOpts = append(logOpts, otlplog.WithHeaders(r.Headers)) + } + } + if logExp, lerr := otlplog.New(ctx, logOpts...); lerr == nil { + lp = sdklog.NewLoggerProvider( + sdklog.WithResource(res), + sdklog.WithProcessor(sdklog.NewBatchProcessor(logExp)), + ) + otellog.SetLoggerProvider(lp) + } + } + return func() { shutdownCtx, cancel := context.WithTimeout(context.Background(), flushTimeout) defer cancel() _ = tp.Shutdown(shutdownCtx) + if lp != nil { + _ = lp.Shutdown(shutdownCtx) + } } } +// logsEnabled reports whether OTEL logs should be exported (enabled with tracing +// unless GITTE_TELEMETRY_LOGS=off). +func logsEnabled() bool { + return !strings.EqualFold(os.Getenv("GITTE_TELEMETRY_LOGS"), "off") +} + // Tracer returns gitte's tracer from the global provider (a no-op tracer when // telemetry is disabled). func Tracer() trace.Tracer { From 328161574a0c1037a67739f5f98ce2477642b0e1 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:03:12 +0200 Subject: [PATCH 14/29] fix(telemetry): resolve log emitter lazily and cover severity/hint mapping --- telemetry/logs.go | 18 +++++- telemetry/logs_test.go | 126 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/telemetry/logs.go b/telemetry/logs.go index 78d9abf7..ad1efdb1 100644 --- a/telemetry/logs.go +++ b/telemetry/logs.go @@ -24,12 +24,14 @@ func NewSpanRegistry() *SpanRegistry { return &SpanRegistry{m: make(map[string]trace.SpanContext)} } +// Set stores the SpanContext sc for task, overwriting any existing entry. func (r *SpanRegistry) Set(task string, sc trace.SpanContext) { r.mu.Lock() r.m[task] = sc r.mu.Unlock() } +// Get returns the SpanContext stored for task and whether it was found. func (r *SpanRegistry) Get(task string) (trace.SpanContext, bool) { r.mu.RLock() sc, ok := r.m[task] @@ -37,6 +39,7 @@ func (r *SpanRegistry) Get(task string) (trace.SpanContext, bool) { return sc, ok } +// Delete removes the entry for task from the registry. func (r *SpanRegistry) Delete(task string) { r.mu.Lock() delete(r.m, task) @@ -47,6 +50,7 @@ func (r *SpanRegistry) Delete(task string) { type logHandler struct { inner executor.OutputHandler reg *SpanRegistry + once sync.Once lgr log.Logger } @@ -54,14 +58,24 @@ type logHandler struct { // OTEL log record correlated (via reg) to the span for output.CmdName. When // logs are disabled the global logger provider is a no-op, so this is safe and // cheap; output is always forwarded to inner unchanged. +// +// The logger is resolved lazily on first use so that callers constructed before +// telemetry.Init registers the real LoggerProvider still pick up the live +// provider. func LogOutputHandler(inner executor.OutputHandler, reg *SpanRegistry) executor.OutputHandler { return &logHandler{ inner: inner, reg: reg, - lgr: global.GetLoggerProvider().Logger("github.com/cego/gitte"), } } +func (h *logHandler) logger() log.Logger { + h.once.Do(func() { + h.lgr = global.GetLoggerProvider().Logger("github.com/cego/gitte") + }) + return h.lgr +} + func (h *logHandler) HandleOutput(ctx context.Context, out executor.Output) error { h.emit(ctx, out) return h.inner.HandleOutput(ctx, out) @@ -87,5 +101,5 @@ func (h *logHandler) emit(ctx context.Context, out executor.Output) { if sc, ok := h.reg.Get(out.CmdName); ok && sc.IsValid() { emitCtx = trace.ContextWithSpanContext(ctx, sc) } - h.lgr.Emit(emitCtx, rec) + h.logger().Emit(emitCtx, rec) } diff --git a/telemetry/logs_test.go b/telemetry/logs_test.go index 61849430..489fd0d5 100644 --- a/telemetry/logs_test.go +++ b/telemetry/logs_test.go @@ -2,10 +2,14 @@ package telemetry import ( "context" + "sync" "testing" "github.com/cego/gitte/executor" + "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/trace" + sdklog "go.opentelemetry.io/otel/sdk/log" ) func TestSpanRegistry_SetGetDelete(t *testing.T) { @@ -48,3 +52,125 @@ func TestLogOutputHandler_ForwardsUnchanged(t *testing.T) { t.Fatalf("inner did not receive line: %+v", inner.lines) } } + +// recordingExporter is a minimal sdklog.Exporter that captures exported records. +type recordingExporter struct { + mu sync.Mutex + records []sdklog.Record +} + +func (e *recordingExporter) Export(_ context.Context, records []sdklog.Record) error { + e.mu.Lock() + defer e.mu.Unlock() + for _, r := range records { + e.records = append(e.records, r.Clone()) + } + return nil +} + +func (e *recordingExporter) Shutdown(_ context.Context) error { return nil } +func (e *recordingExporter) ForceFlush(_ context.Context) error { return nil } + +func (e *recordingExporter) Records() []sdklog.Record { + e.mu.Lock() + defer e.mu.Unlock() + out := make([]sdklog.Record, len(e.records)) + copy(out, e.records) + return out +} + +// setupRecordingProvider installs a real LoggerProvider backed by exp as the +// global, and returns a cleanup function that restores the previous global. +func setupRecordingProvider(t *testing.T) *recordingExporter { + t.Helper() + exp := &recordingExporter{} + proc := sdklog.NewSimpleProcessor(exp) + provider := sdklog.NewLoggerProvider(sdklog.WithProcessor(proc)) + + prev := global.GetLoggerProvider() + global.SetLoggerProvider(provider) + t.Cleanup(func() { + global.SetLoggerProvider(prev) + }) + return exp +} + +func TestLogOutputHandler_StdoutSeverityInfo(t *testing.T) { + exp := setupRecordingProvider(t) + + inner := &recordingHandler{} + reg := NewSpanRegistry() + h := LogOutputHandler(inner, reg) + + _ = h.HandleOutput(context.Background(), executor.Output{ + Output: []byte("normal line"), + CmdName: "task", + Stream: executor.StdoutStream, + }) + + recs := exp.Records() + if len(recs) != 1 { + t.Fatalf("expected 1 record, got %d", len(recs)) + } + if got := recs[0].Severity(); got != log.SeverityInfo { + t.Errorf("stdout severity = %v; want SeverityInfo (%v)", got, log.SeverityInfo) + } +} + +func TestLogOutputHandler_StderrSeverityWarn(t *testing.T) { + exp := setupRecordingProvider(t) + + inner := &recordingHandler{} + reg := NewSpanRegistry() + h := LogOutputHandler(inner, reg) + + _ = h.HandleOutput(context.Background(), executor.Output{ + Output: []byte("error output"), + CmdName: "task", + Stream: executor.StderrStream, + }) + + recs := exp.Records() + if len(recs) != 1 { + t.Fatalf("expected 1 record, got %d", len(recs)) + } + if got := recs[0].Severity(); got != log.SeverityWarn { + t.Errorf("stderr severity = %v; want SeverityWarn (%v)", got, log.SeverityWarn) + } +} + +func TestLogOutputHandler_HintAttribute(t *testing.T) { + exp := setupRecordingProvider(t) + + inner := &recordingHandler{} + reg := NewSpanRegistry() + h := LogOutputHandler(inner, reg) + + _ = h.HandleOutput(context.Background(), executor.Output{ + Output: []byte("[HINT] do something"), + CmdName: "task", + Stream: executor.StdoutStream, + }) + + recs := exp.Records() + if len(recs) != 1 { + t.Fatalf("expected 1 record, got %d", len(recs)) + } + + var hintVal log.Value + var found bool + recs[0].WalkAttributes(func(kv log.KeyValue) bool { + if kv.Key == "gitte.hint" { + hintVal = kv.Value + found = true + return false + } + return true + }) + if !found { + t.Fatal("expected attribute gitte.hint=true but it was not present") + } + if !hintVal.AsBool() { + t.Errorf("gitte.hint = %v; want true", hintVal) + } +} From 6d2d22aff1690a7b36ce1d088f4f2a3d7a8c5f55 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:06:34 +0200 Subject: [PATCH 15/29] feat(telemetry): add phase spans for startup/gitops/actions --- cmd/actions.go | 12 +++++++++++- cmd/gitops.go | 15 +++++++++++++-- cmd/startup.go | 12 +++++++++++- telemetry/telemetry.go | 6 ++++++ telemetry/telemetry_test.go | 11 +++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/cmd/actions.go b/cmd/actions.go index 4d663e79..1b4da467 100644 --- a/cmd/actions.go +++ b/cmd/actions.go @@ -4,6 +4,9 @@ import ( "fmt" "github.com/cego/gitte/actions" + "github.com/cego/gitte/telemetry" + + "go.opentelemetry.io/otel/codes" "github.com/spf13/cobra" ) @@ -38,7 +41,14 @@ func runActions(args []string) error { args[0], actionStr, projectStr, groupStr) } - return actions.RunActions(globalCtx, globalCfg, globalSt, globalCwd, outputMode(), keys, actionOrder, maxParallelization()) + ctx, span := telemetry.StartPhaseSpan(globalCtx, "actions") + defer span.End() + err := actions.RunActions(ctx, globalCfg, globalSt, globalCwd, outputMode(), keys, actionOrder, maxParallelization()) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + return err } // parseActionArgs maps positional CLI args to (actionStr, groupStr, projectStr). diff --git a/cmd/gitops.go b/cmd/gitops.go index be2c0dec..03969897 100644 --- a/cmd/gitops.go +++ b/cmd/gitops.go @@ -9,6 +9,9 @@ import ( "github.com/cego/gitte/gitops" "github.com/cego/gitte/output" + "github.com/cego/gitte/telemetry" + + "go.opentelemetry.io/otel/codes" "github.com/spf13/cobra" ) @@ -41,17 +44,25 @@ SSH concurrency: Discovery clone/pull runs at most 8 SSH connections in parallel to avoid overwhelming the server. Override with GITTE_MAX_TASK_PARALLELIZATION=N.`, RunE: func(cmd *cobra.Command, args []string) error { + ctx, span := telemetry.StartPhaseSpan(globalCtx, "gitops") + defer span.End() mode := outputMode() warnings, addWarning := newWarnCollector() if discover { - if err := gitops.Discover(globalCtx, globalCfg, globalCwd, mode, addWarning); err != nil { + if err := gitops.Discover(ctx, globalCfg, globalCwd, mode, addWarning); err != nil { gitops.PrintWarnings(mode, warnings()) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return err } } nr := noRebase || os.Getenv("GITTE_NO_REBASE") == "true" - err := gitops.Sync(globalCtx, globalCfg, globalCwd, mode, nr, makePromptFn(mode), addWarning) + err := gitops.Sync(ctx, globalCfg, globalCwd, mode, nr, makePromptFn(mode), addWarning) gitops.PrintWarnings(mode, warnings()) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } return err }, } diff --git a/cmd/startup.go b/cmd/startup.go index efd95f49..fa0f0cd3 100644 --- a/cmd/startup.go +++ b/cmd/startup.go @@ -2,6 +2,9 @@ package cmd import ( "github.com/cego/gitte/startup" + "github.com/cego/gitte/telemetry" + + "go.opentelemetry.io/otel/codes" "github.com/spf13/cobra" ) @@ -11,7 +14,14 @@ func newStartupCmd() *cobra.Command { Use: "startup", Short: "Run startup checks", RunE: func(cmd *cobra.Command, args []string) error { - return startup.Run(globalCtx, globalCfg, globalCwd, outputMode()) + ctx, span := telemetry.StartPhaseSpan(globalCtx, "startup") + defer span.End() + err := startup.Run(ctx, globalCfg, globalCwd, outputMode()) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + return err }, } } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index acd5efaa..5333cda9 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -195,6 +195,12 @@ func Tracer() trace.Tracer { return otel.Tracer(tracerName) } +// StartPhaseSpan starts a span for a gitte run phase (startup/gitops/actions) +// and returns the derived context to thread into that phase's work. +func StartPhaseSpan(ctx context.Context, phase string) (context.Context, trace.Span) { + return Tracer().Start(ctx, phase) +} + // StartCommandSpan starts the root span for a gitte invocation. func StartCommandSpan(ctx context.Context, commandPath string, args []string) (context.Context, trace.Span) { ctx, span := Tracer().Start(ctx, commandPath) diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index b239916c..12f6e4fa 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -135,3 +135,14 @@ func TestStartCommandSpan_NoProviderDoesNotPanic(t *testing.T) { _, span := StartCommandSpan(context.Background(), "gitte run", []string{"up"}) span.End() } + +func TestStartPhaseSpan_ReturnsChildContext(t *testing.T) { + ctx, span := StartPhaseSpan(context.Background(), "startup") + if span == nil { + t.Fatal("nil span") + } + if ctx == context.Background() { + t.Fatal("expected a derived context") + } + span.End() +} From d1b64aabe9333191cfff24193a3e8d5c6ee7f7d2 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:09:19 +0200 Subject: [PATCH 16/29] feat(cmd): wrap gitte run phases in startup/gitops/actions spans --- cmd/run.go | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 9da1772c..9b1b8f42 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -6,6 +6,8 @@ import ( "github.com/cego/gitte/gitops" "github.com/cego/gitte/startup" + "github.com/cego/gitte/telemetry" + "go.opentelemetry.io/otel/codes" "github.com/spf13/cobra" ) @@ -28,33 +30,50 @@ Examples: ValidArgsFunction: actionArgsCompletion, RunE: func(cmd *cobra.Command, args []string) error { // Step 1: Startup checks - if err := startup.Run(globalCtx, globalCfg, globalCwd, outputMode()); err != nil { - return err + startupCtx, startupSpan := telemetry.StartPhaseSpan(globalCtx, "startup") + serr := startup.Run(startupCtx, globalCfg, globalCwd, outputMode()) + if serr != nil { + startupSpan.RecordError(serr) + startupSpan.SetStatus(codes.Error, serr.Error()) + } + startupSpan.End() + if serr != nil { + return serr } fmt.Println() - // Step 2: Discovery (if requested) + // Step 2: Discovery + git sync mode := outputMode() warnings, addWarning := newWarnCollector() - if discover { - if err := gitops.Discover(globalCtx, globalCfg, globalCwd, mode, addWarning); err != nil { + gitopsCtx, gitopsSpan := telemetry.StartPhaseSpan(globalCtx, "gitops") + gerr := func() error { + if discover { + if err := gitops.Discover(gitopsCtx, globalCfg, globalCwd, mode, addWarning); err != nil { + gitops.PrintWarnings(mode, warnings()) + return err + } + } + nr := noRebase || os.Getenv("GITTE_NO_REBASE") == "true" + if err := gitops.Sync(gitopsCtx, globalCfg, globalCwd, mode, nr, makePromptFn(mode), addWarning); err != nil { gitops.PrintWarnings(mode, warnings()) return err } - } - - // Step 3: Git sync - nr := noRebase || os.Getenv("GITTE_NO_REBASE") == "true" - if err := gitops.Sync(globalCtx, globalCfg, globalCwd, mode, nr, makePromptFn(mode), addWarning); err != nil { gitops.PrintWarnings(mode, warnings()) - return err + return nil + }() + if gerr != nil { + gitopsSpan.RecordError(gerr) + gitopsSpan.SetStatus(codes.Error, gerr.Error()) + } + gitopsSpan.End() + if gerr != nil { + return gerr } - gitops.PrintWarnings(mode, warnings()) fmt.Println() - // Step 4: Actions (if specified) + // Step 3: Actions (if specified) — runActions opens its own "actions" phase span. if len(args) > 0 { return runActions(args) } From fde5666a740bd5ca6d825803b46e6a38fdcbd6bd Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:13:38 +0200 Subject: [PATCH 17/29] feat(startup): add per-check spans and ship check output as logs --- startup/startup.go | 28 ++++++++++++++++++++++++---- startup/startup_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 startup/startup_test.go diff --git a/startup/startup.go b/startup/startup.go index cf00dcb5..064a2b76 100644 --- a/startup/startup.go +++ b/startup/startup.go @@ -9,6 +9,9 @@ import ( "github.com/cego/gitte/config" "github.com/cego/gitte/executor" "github.com/cego/gitte/output" + "github.com/cego/gitte/telemetry" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) // Run executes all startup checks and streams status to stdout. @@ -21,6 +24,7 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O ctx, cancel := context.WithCancel(ctx) defer cancel() + reg := telemetry.NewSpanRegistry() tasks := make([]executor.Task, 0, len(cfg.StartupChecks)) for name, check := range cfg.StartupChecks { name := name @@ -28,13 +32,23 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O tasks = append(tasks, executor.Task{ Name: name, Needs: check.GetNeeds(), - ExecuteFn: func(ctx context.Context, taskName string, handler executor.OutputHandler) error { - if err := check.Check(ctx, cwd); err != nil { + ExecuteFn: func(ctx context.Context, taskName string, handler executor.OutputHandler) (err error) { + ctx, span := startCheckSpan(ctx, taskName) + reg.Set(taskName, span.SpanContext()) + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + reg.Delete(taskName) + }() + if cerr := check.Check(ctx, cwd); cerr != nil { hint := check.GetHint() if hint != "" { - return fmt.Errorf("%s\nhint: %s", err.Error(), hint) + return fmt.Errorf("%s\nhint: %s", cerr.Error(), hint) } - return err + return cerr } return nil }, @@ -51,6 +65,7 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O if err != nil { return fmt.Errorf("startup checks have invalid dependencies: %w", err) } + exec.WithOutputHandler(telemetry.LogOutputHandler(executor.NoopOutputHandler{}, reg)) runErr := exec.Execute(ctx) view.Wait() @@ -62,6 +77,11 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O return runErr } +// startCheckSpan opens a span for a single startup check. +func startCheckSpan(ctx context.Context, name string) (context.Context, trace.Span) { + return telemetry.Tracer().Start(ctx, "startup.check "+name) +} + // newView picks the right view implementation based on output mode. func newView(mode output.OutputMode, tasks []executor.Task, cancel context.CancelFunc) View { if mode == output.ModePlain { diff --git a/startup/startup_test.go b/startup/startup_test.go new file mode 100644 index 00000000..1bab0bbf --- /dev/null +++ b/startup/startup_test.go @@ -0,0 +1,27 @@ +package startup + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestStartCheckSpan_RecordsNamedSpan(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + prev := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { otel.SetTracerProvider(prev); _ = tp.Shutdown(context.Background()) }) + + ctx, span := startCheckSpan(context.Background(), "git-present") + span.End() + _ = ctx + + spans := exp.GetSpans() + if len(spans) != 1 || spans[0].Name != "startup.check git-present" { + t.Fatalf("got %+v, want one span named 'startup.check git-present'", spans) + } +} From d12961c7306d194f43b0e1ae2059ec48b02ac5dd Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:19:18 +0200 Subject: [PATCH 18/29] feat(actions): add per-action spans, parent task spans, ship action output as logs --- actions/runner.go | 26 ++++++++--- telemetry/action_tracker.go | 79 ++++++++++++++++++++++++++++++++ telemetry/action_tracker_test.go | 34 ++++++++++++++ 3 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 telemetry/action_tracker.go create mode 100644 telemetry/action_tracker_test.go diff --git a/actions/runner.go b/actions/runner.go index 7fc4f1d3..4e3fad22 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -39,6 +39,14 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta infos := buildTaskInfos(cfg, st, cwd, keys) view := newView(mode, infos, actionOrder, runCancel, retryCh, cfg.QuickSolve.GitClean.Exclude) + tracker := telemetry.NewActionTracker(ctx) + reg := telemetry.NewSpanRegistry() + + onStart := func(name string) { + tracker.OnStart(name) + view.OnStart(name) + } + // Track per-task outcomes so retry runs can pre-complete tasks that already finished. outcomes := newTaskOutcomes() onFinish := func(name string, err error, elapsed time.Duration) { @@ -50,6 +58,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta outcomes.set(name, outcomeFailed) } view.OnFinish(name, err, elapsed) + tracker.OnFinish(name) } maxParallel := envMaxParallel @@ -63,7 +72,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta var retrySet map[string]struct{} // nil on first run var runErr error for { - tasks := buildExecutorTasks(cfg, st, cwd, keys) + tasks := buildExecutorTasks(cfg, st, cwd, keys, tracker, reg) // Strip needs from explicitly retried tasks so they run immediately. if retrySet != nil { @@ -76,7 +85,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta exec, err := executor.NewExecutor(tasks, executor.ExecutorOptions{ MaxParallelization: maxParallel, - OnTaskStart: view.OnStart, + OnTaskStart: onStart, OnTaskReset: view.OnReset, OnTaskFinish: onFinish, }) @@ -105,7 +114,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta exec.WithPreCompleted(succeeded, failed) } - exec.WithOutputHandler(view.Handler()) + exec.WithOutputHandler(telemetry.LogOutputHandler(view.Handler(), reg)) exec.WithRetryChannel(retryCh) runErr = exec.Execute(runCtx) @@ -199,7 +208,7 @@ func buildTaskInfos(cfg *config.GitteConfig, st *state.GitteState, cwd string, k } // buildExecutorTasks constructs executor.Task list from keys. -func buildExecutorTasks(cfg *config.GitteConfig, st *state.GitteState, cwd string, keys []GroupKeyWithDeps) []executor.Task { +func buildExecutorTasks(cfg *config.GitteConfig, st *state.GitteState, cwd string, keys []GroupKeyWithDeps, tracker *telemetry.ActionTracker, reg *telemetry.SpanRegistry) []executor.Task { tasks := make([]executor.Task, 0, len(keys)) searchFors := cfg.SearchFor @@ -244,7 +253,7 @@ func buildExecutorTasks(cfg *config.GitteConfig, st *state.GitteState, cwd strin Needs: needNames, Retry: retryConfig, ExecuteFn: func(ctx context.Context, tName string, handler executor.OutputHandler) error { - return runGroupTask(ctx, cfg, st, cwd, proj, key.Project, tName, cmds, allSearchFors, handler) + return runGroupTask(ctx, cfg, st, cwd, proj, key.Project, tName, cmds, allSearchFors, handler, tracker, reg) }, }) } @@ -275,8 +284,12 @@ func runGroupTask( cmds []string, searchFors []config.SearchFor, handler executor.OutputHandler, + tracker *telemetry.ActionTracker, + reg *telemetry.SpanRegistry, ) (err error) { - ctx, span := telemetry.Tracer().Start(ctx, "action.run") + actionCtx := tracker.ActionContext(telemetry.ActionOf(taskName)) + ctx, span := telemetry.Tracer().Start(actionCtx, "action.run") + reg.Set(taskName, span.SpanContext()) setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) defer func() { if err != nil { @@ -284,6 +297,7 @@ func runGroupTask( span.SetStatus(codes.Error, err.Error()) } span.End() + reg.Delete(taskName) }() if len(cmds) == 0 { diff --git a/telemetry/action_tracker.go b/telemetry/action_tracker.go new file mode 100644 index 00000000..58a9e95b --- /dev/null +++ b/telemetry/action_tracker.go @@ -0,0 +1,79 @@ +package telemetry + +import ( + "context" + "strings" + "sync" + + "go.opentelemetry.io/otel/trace" +) + +// ActionTracker opens one span per action (e.g. "build", "up") under the +// actions phase context, driven by the executor's task hooks. An action span +// opens on its first task start and closes when its last task finishes. +type ActionTracker struct { + phaseCtx context.Context + mu sync.Mutex + spans map[string]trace.Span // action -> span + ctxs map[string]context.Context // action -> span context + active map[string]int // action -> live task count +} + +// NewActionTracker creates a tracker rooted at the actions phase context. +func NewActionTracker(phaseCtx context.Context) *ActionTracker { + return &ActionTracker{ + phaseCtx: phaseCtx, + spans: map[string]trace.Span{}, + ctxs: map[string]context.Context{}, + active: map[string]int{}, + } +} + +// ActionOf extracts the action name from a "project:action:group" task name. +func ActionOf(taskName string) string { + parts := strings.Split(taskName, ":") + if len(parts) >= 2 { + return parts[1] // project:action:group + } + return taskName +} + +// OnStart opens the action span if needed and increments its live-task count. +func (t *ActionTracker) OnStart(taskName string) { + action := ActionOf(taskName) + t.mu.Lock() + defer t.mu.Unlock() + if _, ok := t.spans[action]; !ok { + ctx, span := Tracer().Start(t.phaseCtx, action) + t.spans[action] = span + t.ctxs[action] = ctx + } + t.active[action]++ +} + +// OnFinish decrements the live-task count and ends the action span at zero. +func (t *ActionTracker) OnFinish(taskName string) { + action := ActionOf(taskName) + t.mu.Lock() + defer t.mu.Unlock() + t.active[action]-- + if t.active[action] <= 0 { + if span, ok := t.spans[action]; ok { + span.End() + delete(t.spans, action) + delete(t.ctxs, action) + delete(t.active, action) + } + } +} + +// ActionContext returns the action span's context, or the phase context if the +// action span is not open. +func (t *ActionTracker) ActionContext(action string) context.Context { + t.mu.Lock() + defer t.mu.Unlock() + if ctx, ok := t.ctxs[action]; ok { + return ctx + } + return t.phaseCtx +} diff --git a/telemetry/action_tracker_test.go b/telemetry/action_tracker_test.go new file mode 100644 index 00000000..0d89c1ab --- /dev/null +++ b/telemetry/action_tracker_test.go @@ -0,0 +1,34 @@ +package telemetry + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestActionTracker_SpanPerAction(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + prev := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { otel.SetTracerProvider(prev); _ = tp.Shutdown(context.Background()) }) + + tr := NewActionTracker(context.Background()) + tr.OnStart("a:build:sn") + tr.OnStart("b:build:sn") + tr.OnFinish("a:build:sn") + tr.OnFinish("b:build:sn") // last build task -> build span ends + tr.OnStart("a:up:sn") + tr.OnFinish("a:up:sn") // up span ends + + names := map[string]int{} + for _, s := range exp.GetSpans() { + names[s.Name]++ + } + if names["build"] != 1 || names["up"] != 1 { + t.Fatalf("want one build and one up span, got %v", names) + } +} From 40c22aa263dd4f5c814d0e578e929e37c84c35b6 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:23:41 +0200 Subject: [PATCH 19/29] fix(telemetry): ignore OnFinish for never-started (skipped) tasks in ActionTracker --- telemetry/action_tracker.go | 11 ++++++- telemetry/action_tracker_test.go | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/telemetry/action_tracker.go b/telemetry/action_tracker.go index 58a9e95b..4e6e154b 100644 --- a/telemetry/action_tracker.go +++ b/telemetry/action_tracker.go @@ -17,6 +17,7 @@ type ActionTracker struct { spans map[string]trace.Span // action -> span ctxs map[string]context.Context // action -> span context active map[string]int // action -> live task count + started map[string]struct{} // tasks that have called OnStart } // NewActionTracker creates a tracker rooted at the actions phase context. @@ -26,6 +27,7 @@ func NewActionTracker(phaseCtx context.Context) *ActionTracker { spans: map[string]trace.Span{}, ctxs: map[string]context.Context{}, active: map[string]int{}, + started: map[string]struct{}{}, } } @@ -48,14 +50,21 @@ func (t *ActionTracker) OnStart(taskName string) { t.spans[action] = span t.ctxs[action] = ctx } + t.started[taskName] = struct{}{} t.active[action]++ } // OnFinish decrements the live-task count and ends the action span at zero. +// If the task never called OnStart (e.g. it was skipped due to a failed +// dependency), this is a no-op to avoid corrupting the active count. func (t *ActionTracker) OnFinish(taskName string) { - action := ActionOf(taskName) t.mu.Lock() defer t.mu.Unlock() + if _, ok := t.started[taskName]; !ok { + return // skipped task — no matching OnStart, nothing to do + } + delete(t.started, taskName) + action := ActionOf(taskName) t.active[action]-- if t.active[action] <= 0 { if span, ok := t.spans[action]; ok { diff --git a/telemetry/action_tracker_test.go b/telemetry/action_tracker_test.go index 0d89c1ab..57228b7b 100644 --- a/telemetry/action_tracker_test.go +++ b/telemetry/action_tracker_test.go @@ -32,3 +32,57 @@ func TestActionTracker_SpanPerAction(t *testing.T) { t.Fatalf("want one build and one up span, got %v", names) } } + +func TestActionTracker_SkippedTaskDoesNotCloseSpan(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + prev := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { otel.SetTracerProvider(prev); _ = tp.Shutdown(context.Background()) }) + + tr := NewActionTracker(context.Background()) + + // Two real tasks start. + tr.OnStart("a:build:sn") + tr.OnStart("b:build:sn") + + // A skipped task (never started) fires OnFinish — must be a no-op. + tr.OnFinish("c:build:sn") + + // Action span must still be open: ActionContext returns the action's own + // context (not the phase/background context), and no "build" span exported. + ctxAfterSkip := tr.ActionContext("build") + if ctxAfterSkip == context.Background() { + t.Fatal("action span was prematurely closed by skipped task's OnFinish") + } + if n := countSpans(exp, "build"); n != 0 { + t.Fatalf("want 0 exported build spans after skipped OnFinish, got %d", n) + } + + // One real task finishes — span still open because b:build:sn is active. + tr.OnFinish("a:build:sn") + if n := countSpans(exp, "build"); n != 0 { + t.Fatalf("want 0 exported build spans after first real OnFinish, got %d", n) + } + + // Last real task finishes — span must close now, exactly once. + tr.OnFinish("b:build:sn") + if n := countSpans(exp, "build"); n != 1 { + t.Fatalf("want exactly 1 exported build span after last real OnFinish, got %d", n) + } + + // ActionContext must now fall back to the phase context. + if tr.ActionContext("build") != context.Background() { + t.Fatal("expected action context to fall back to phase context after span closed") + } +} + +func countSpans(exp *tracetest.InMemoryExporter, name string) int { + n := 0 + for _, s := range exp.GetSpans() { + if s.Name == name { + n++ + } + } + return n +} From 8dd28b827b8c425ac1540e4db42f43ba18da5aa3 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:26:17 +0200 Subject: [PATCH 20/29] docs: document span hierarchy and OTEL log export --- README.md | 11 +++++++++++ docs/config.md | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 74912e9d..8e920c3a 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,16 @@ point). Keep secrets out of action command definitions and CLI arguments; pass them through environment variables, which are **not** exported. Full remote URLs are never collected either (repos are identified by name only). +Each `gitte run` produces a structured trace: a root span with child spans for +each phase (`startup`, `gitops`, `actions`), a span per startup check, a span +per action (e.g. `build`, `up`) parenting its task spans, and a span per repo +sync. Action and startup command output is also shipped as **OTEL logs**, +correlated to the span that produced each line (stdout → INFO, stderr → WARN). + +Logs are enabled with tracing; set `GITTE_TELEMETRY_LOGS=off` to keep traces but +disable the (higher-volume) log export. Keep secrets out of command output — +log lines are exported verbatim. + Enable it via the shared config: ```yaml @@ -200,6 +210,7 @@ Environment variables: |---|---| | `GITTE_TELEMETRY=off` | Disable telemetry locally (kill-switch) | | `GITTE_TELEMETRY_URL` | Override the endpoint | +| `GITTE_TELEMETRY_LOGS=off` | Disable OTEL log export (keep traces) | | `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` | Standard OTEL env vars, honored as a fallback when no gitte endpoint is set | Precedence: `GITTE_TELEMETRY=off` > `GITTE_TELEMETRY_URL` > config endpoint > diff --git a/docs/config.md b/docs/config.md index 5b888d63..70c2cfd6 100644 --- a/docs/config.md +++ b/docs/config.md @@ -434,10 +434,14 @@ Environment variables override or disable telemetry: |----------|--------| | `GITTE_TELEMETRY=off` | Disable telemetry locally (kill-switch) | | `GITTE_TELEMETRY_URL` | Override the endpoint | +| `GITTE_TELEMETRY_LOGS=off` | Disable OTEL log export (keep traces) | | `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` | Standard OTEL env vars, honored as a fallback when no gitte endpoint is set | Precedence: `GITTE_TELEMETRY=off` > `GITTE_TELEMETRY_URL` > config `endpoint` > `OTEL_EXPORTER_OTLP_*`. Telemetry is best-effort and never blocks or slows gitte; export failures are silently ignored and flushing on exit is time-bounded. +Action and startup command output is also exported as OTEL logs (correlated to +the producing span) unless `GITTE_TELEMETRY_LOGS=off`. + --- ## Remote configuration From 0673b63683a956f945d8c4ce83dc8e6e5b9a05a3 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:38:33 +0200 Subject: [PATCH 21/29] fix(telemetry): target /v1/logs for OTLP log export (was POSTing to server root) --- telemetry/logs_test.go | 14 ++++++++++++++ telemetry/telemetry.go | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/telemetry/logs_test.go b/telemetry/logs_test.go index 489fd0d5..057687bc 100644 --- a/telemetry/logs_test.go +++ b/telemetry/logs_test.go @@ -174,3 +174,17 @@ func TestLogOutputHandler_HintAttribute(t *testing.T) { t.Errorf("gitte.hint = %v; want true", hintVal) } } + +func TestLogsEndpointURL(t *testing.T) { + cases := []struct{ in, want string }{ + {"https://apm.example.com", "https://apm.example.com/v1/logs"}, + {"https://apm.example.com/", "https://apm.example.com/v1/logs"}, + {"https://apm.example.com:8200", "https://apm.example.com:8200/v1/logs"}, + {"https://apm.example.com/custom/logs", "https://apm.example.com/custom/logs"}, + } + for _, c := range cases { + if got := logsEndpointURL(c.in); got != c.want { + t.Errorf("logsEndpointURL(%q) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 5333cda9..dce1a12d 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -7,6 +7,7 @@ package telemetry import ( "context" "fmt" + "net/url" "os" "os/user" "runtime" @@ -159,7 +160,7 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { if logsEnabled() { var logOpts []otlplog.Option if !r.UseSDKEnv { - logOpts = append(logOpts, otlplog.WithEndpointURL(r.Endpoint)) + logOpts = append(logOpts, otlplog.WithEndpointURL(logsEndpointURL(r.Endpoint))) if len(r.Headers) > 0 { logOpts = append(logOpts, otlplog.WithHeaders(r.Headers)) } @@ -189,6 +190,20 @@ func logsEnabled() bool { return !strings.EqualFold(os.Getenv("GITTE_TELEMETRY_LOGS"), "off") } +// logsEndpointURL returns the OTLP/HTTP logs endpoint for a configured base +// endpoint. otlploghttp.WithEndpointURL uses the URL's path verbatim, so a +// path-less endpoint (e.g. "https://apm.example.com") would POST to the server +// root and be rejected. When the endpoint has no path we append the standard +// "/v1/logs" intake path (matching how the traces exporter targets +// "/v1/traces"); an endpoint that already carries a path is left untouched. +func logsEndpointURL(endpoint string) string { + u, err := url.Parse(endpoint) + if err != nil || u.Path == "" || u.Path == "/" { + return strings.TrimRight(endpoint, "/") + "/v1/logs" + } + return endpoint +} + // Tracer returns gitte's tracer from the global provider (a no-op tracer when // telemetry is disabled). func Tracer() trace.Tracer { From c0d5b8d17df8f282a19140d07c334d34adf8ef6d Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 12:58:20 +0200 Subject: [PATCH 22/29] feat(telemetry): include repo/task in gitops.sync and action.run span names --- actions/runner.go | 2 +- gitops/gitops.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/runner.go b/actions/runner.go index 4e3fad22..599b96f8 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -288,7 +288,7 @@ func runGroupTask( reg *telemetry.SpanRegistry, ) (err error) { actionCtx := tracker.ActionContext(telemetry.ActionOf(taskName)) - ctx, span := telemetry.Tracer().Start(actionCtx, "action.run") + ctx, span := telemetry.Tracer().Start(actionCtx, "action.run "+taskName) reg.Set(taskName, span.SpanContext()) setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) defer func() { diff --git a/gitops/gitops.go b/gitops/gitops.go index bf5272aa..4ab32ae9 100644 --- a/gitops/gitops.go +++ b/gitops/gitops.go @@ -188,7 +188,7 @@ func syncProject( addPrompt func(CheckoutPrompt), warnFn func(string), ) (err error) { - ctx, span := telemetry.Tracer().Start(ctx, "gitops.sync") + ctx, span := telemetry.Tracer().Start(ctx, "gitops.sync "+name) span.SetAttributes(attribute.String("gitte.repo", name)) defer func() { if err != nil { From e77aadd9ac691654763aa6d21cdedf8baa2b770b Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 13:43:52 +0200 Subject: [PATCH 23/29] feat(telemetry): label action task spans with enabled feature gates --- actions/features_test.go | 31 +++++++++++++++++++++++++++++++ actions/runner.go | 31 ++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 actions/features_test.go diff --git a/actions/features_test.go b/actions/features_test.go new file mode 100644 index 00000000..9d21ee8f --- /dev/null +++ b/actions/features_test.go @@ -0,0 +1,31 @@ +package actions + +import ( + "reflect" + "testing" + + "github.com/cego/gitte/config" + "github.com/cego/gitte/state" +) + +func TestEnabledFeaturesForProject(t *testing.T) { + cfg := &config.GitteConfig{ + FeatureGates: map[string]config.FeatureGate{ + "feat-on": {}, // empty scope → applies to all projects + "feat-off": {}, // disabled in state + "feat-scoped-out": {Scope: config.FeatureScope{Projects: []string{"other"}}}, // enabled but scoped to a different project + }, + } + st := &state.GitteState{Features: map[string]state.FeatureState{ + "feat-on": {Enabled: true}, + "feat-off": {Enabled: false}, + "feat-scoped-out": {Enabled: true}, + }} + proj := config.ProjectConfig{Remote: "git@github.com:example/myproj.git"} + + got := enabledFeaturesForProject(cfg, st, "myproj", proj) + want := []string{"feat-on"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("enabledFeaturesForProject = %v, want %v", got, want) + } +} diff --git a/actions/runner.go b/actions/runner.go index 599b96f8..6b0ac066 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -291,6 +291,9 @@ func runGroupTask( ctx, span := telemetry.Tracer().Start(actionCtx, "action.run "+taskName) reg.Set(taskName, span.SpanContext()) setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) + if feats := enabledFeaturesForProject(cfg, st, projName, proj); len(feats) > 0 { + span.SetAttributes(attribute.StringSlice("gitte.features", feats)) + } defer func() { if err != nil { span.RecordError(err) @@ -388,13 +391,15 @@ func emitTaskPreamble( } } -// extraEnvForProject returns the env vars injected by feature gates for a project. -func extraEnvForProject(cfg *config.GitteConfig, st *state.GitteState, projName string, proj config.ProjectConfig) map[string]string { +// enabledFeaturesForProject returns the sorted names of feature gates that are +// enabled and in scope for the given project (the gates that actually inject +// env into the project's tasks). +func enabledFeaturesForProject(cfg *config.GitteConfig, st *state.GitteState, projName string, proj config.ProjectConfig) []string { if st == nil || cfg.FeatureGates == nil { return nil } - extra := make(map[string]string) + var names []string for gateName, gate := range cfg.FeatureGates { fs, enabled := st.Features[gateName] if !enabled || !fs.Enabled { @@ -417,6 +422,22 @@ func extraEnvForProject(cfg *config.GitteConfig, st *state.GitteState, projName } } + names = append(names, gateName) + } + sort.Strings(names) + return names +} + +// extraEnvForProject returns the env vars injected by feature gates for a project. +func extraEnvForProject(cfg *config.GitteConfig, st *state.GitteState, projName string, proj config.ProjectConfig) map[string]string { + gates := enabledFeaturesForProject(cfg, st, projName, proj) + if len(gates) == 0 { + return nil + } + + extra := make(map[string]string) + for _, gateName := range gates { + gate := cfg.FeatureGates[gateName] for k, v := range gate.Effects.Env { extra[k] = v } @@ -424,10 +445,6 @@ func extraEnvForProject(cfg *config.GitteConfig, st *state.GitteState, projName extra[k] = v } } - - if len(extra) == 0 { - return nil - } return extra } From c470389ccd1f77b4038d1daa28b05dc72dc35bc1 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Fri, 26 Jun 2026 13:48:03 +0200 Subject: [PATCH 24/29] feat(telemetry): add gitte.env (injected env KEY=VALUE) to action task spans --- actions/features_test.go | 17 +++++++++++++++++ actions/runner.go | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/actions/features_test.go b/actions/features_test.go index 9d21ee8f..80fbaa89 100644 --- a/actions/features_test.go +++ b/actions/features_test.go @@ -29,3 +29,20 @@ func TestEnabledFeaturesForProject(t *testing.T) { t.Fatalf("enabledFeaturesForProject = %v, want %v", got, want) } } + +func TestInjectedEnv(t *testing.T) { + cfg := &config.GitteConfig{ + FeatureGates: map[string]config.FeatureGate{ + "feat-on": {Effects: config.FeatureEffects{Env: map[string]string{"FEAT_VAR": "1"}}}, + }, + } + st := &state.GitteState{Features: map[string]state.FeatureState{"feat-on": {Enabled: true}}} + proj := config.ProjectConfig{ + Remote: "git@github.com:example/myproj.git", + Env: map[string]string{"PROJ_VAR": "x"}, + } + got := injectedEnv(cfg, st, "myproj", proj) + if got["PROJ_VAR"] != "x" || got["FEAT_VAR"] != "1" { + t.Fatalf("injectedEnv = %v, want PROJ_VAR=x and FEAT_VAR=1", got) + } +} diff --git a/actions/runner.go b/actions/runner.go index 6b0ac066..3bd5098e 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -294,6 +294,18 @@ func runGroupTask( if feats := enabledFeaturesForProject(cfg, st, projName, proj); len(feats) > 0 { span.SetAttributes(attribute.StringSlice("gitte.features", feats)) } + if env := injectedEnv(cfg, st, projName, proj); len(env) > 0 { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + kvs := make([]string, 0, len(keys)) + for _, k := range keys { + kvs = append(kvs, k+"="+env[k]) + } + span.SetAttributes(attribute.StringSlice("gitte.env", kvs)) + } defer func() { if err != nil { span.RecordError(err) @@ -367,16 +379,7 @@ func emitTaskPreamble( emit(" cmd: " + strings.Join(cmds, " ")) // Collect only the vars gitte injects (not all of os.Environ). - injected := make(map[string]string) - for k, v := range proj.Env { - injected[k] = v - } - for k, v := range config.ResolveEnvWhen(proj.EnvWhen, runtime.GOARCH) { - injected[k] = v - } - for k, v := range extraEnvForProject(cfg, st, projName, proj) { - injected[k] = v - } + injected := injectedEnv(cfg, st, projName, proj) if len(injected) > 0 { keys := make([]string, 0, len(injected)) for k := range injected { @@ -391,6 +394,23 @@ func emitTaskPreamble( } } +// injectedEnv returns the env vars gitte injects for a project's task — project +// env, arch-conditional env_when, and enabled feature-gate env — excluding the +// inherited process environment (os.Environ). +func injectedEnv(cfg *config.GitteConfig, st *state.GitteState, projName string, proj config.ProjectConfig) map[string]string { + injected := make(map[string]string) + for k, v := range proj.Env { + injected[k] = v + } + for k, v := range config.ResolveEnvWhen(proj.EnvWhen, runtime.GOARCH) { + injected[k] = v + } + for k, v := range extraEnvForProject(cfg, st, projName, proj) { + injected[k] = v + } + return injected +} + // enabledFeaturesForProject returns the sorted names of feature gates that are // enabled and in scope for the given project (the gates that actually inject // env into the project's tasks). From 0f93c1fbc0b1c9283e15aa3a8b13383ca072e8ac Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Mon, 29 Jun 2026 10:27:00 +0200 Subject: [PATCH 25/29] fix(telemetry): record task errors on the action span, not just the task/root --- actions/runner.go | 2 +- telemetry/action_tracker.go | 16 +++++++++--- telemetry/action_tracker_test.go | 42 +++++++++++++++++++++++++++----- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/actions/runner.go b/actions/runner.go index 3bd5098e..e543d93e 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -58,7 +58,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta outcomes.set(name, outcomeFailed) } view.OnFinish(name, err, elapsed) - tracker.OnFinish(name) + tracker.OnFinish(name, err) } maxParallel := envMaxParallel diff --git a/telemetry/action_tracker.go b/telemetry/action_tracker.go index 4e6e154b..6eef1ade 100644 --- a/telemetry/action_tracker.go +++ b/telemetry/action_tracker.go @@ -5,6 +5,7 @@ import ( "strings" "sync" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" ) @@ -55,9 +56,10 @@ func (t *ActionTracker) OnStart(taskName string) { } // OnFinish decrements the live-task count and ends the action span at zero. -// If the task never called OnStart (e.g. it was skipped due to a failed -// dependency), this is a no-op to avoid corrupting the active count. -func (t *ActionTracker) OnFinish(taskName string) { +// A non-nil err is recorded on the action span so failure surfaces at the +// action level too. If the task never called OnStart (e.g. it was skipped due +// to a failed dependency), this is a no-op to avoid corrupting the active count. +func (t *ActionTracker) OnFinish(taskName string, err error) { t.mu.Lock() defer t.mu.Unlock() if _, ok := t.started[taskName]; !ok { @@ -65,6 +67,14 @@ func (t *ActionTracker) OnFinish(taskName string) { } delete(t.started, taskName) action := ActionOf(taskName) + // Propagate a failed task onto its action span so failure shows at every + // level of the trace, not just on the task span. + if err != nil { + if span, ok := t.spans[action]; ok { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + } t.active[action]-- if t.active[action] <= 0 { if span, ok := t.spans[action]; ok { diff --git a/telemetry/action_tracker_test.go b/telemetry/action_tracker_test.go index 57228b7b..d4658380 100644 --- a/telemetry/action_tracker_test.go +++ b/telemetry/action_tracker_test.go @@ -2,9 +2,11 @@ package telemetry import ( "context" + "errors" "testing" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" ) @@ -19,10 +21,10 @@ func TestActionTracker_SpanPerAction(t *testing.T) { tr := NewActionTracker(context.Background()) tr.OnStart("a:build:sn") tr.OnStart("b:build:sn") - tr.OnFinish("a:build:sn") - tr.OnFinish("b:build:sn") // last build task -> build span ends + tr.OnFinish("a:build:sn", nil) + tr.OnFinish("b:build:sn", nil) // last build task -> build span ends tr.OnStart("a:up:sn") - tr.OnFinish("a:up:sn") // up span ends + tr.OnFinish("a:up:sn", nil) // up span ends names := map[string]int{} for _, s := range exp.GetSpans() { @@ -47,7 +49,7 @@ func TestActionTracker_SkippedTaskDoesNotCloseSpan(t *testing.T) { tr.OnStart("b:build:sn") // A skipped task (never started) fires OnFinish — must be a no-op. - tr.OnFinish("c:build:sn") + tr.OnFinish("c:build:sn", nil) // Action span must still be open: ActionContext returns the action's own // context (not the phase/background context), and no "build" span exported. @@ -60,13 +62,13 @@ func TestActionTracker_SkippedTaskDoesNotCloseSpan(t *testing.T) { } // One real task finishes — span still open because b:build:sn is active. - tr.OnFinish("a:build:sn") + tr.OnFinish("a:build:sn", nil) if n := countSpans(exp, "build"); n != 0 { t.Fatalf("want 0 exported build spans after first real OnFinish, got %d", n) } // Last real task finishes — span must close now, exactly once. - tr.OnFinish("b:build:sn") + tr.OnFinish("b:build:sn", nil) if n := countSpans(exp, "build"); n != 1 { t.Fatalf("want exactly 1 exported build span after last real OnFinish, got %d", n) } @@ -86,3 +88,31 @@ func countSpans(exp *tracetest.InMemoryExporter, name string) int { } return n } + +func TestActionTracker_RecordsTaskErrorOnActionSpan(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + prev := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { otel.SetTracerProvider(prev); _ = tp.Shutdown(context.Background()) }) + + tr := NewActionTracker(context.Background()) + tr.OnStart("a:build:sn") + tr.OnStart("b:build:sn") + tr.OnFinish("a:build:sn", errors.New("build failed")) // one task fails + tr.OnFinish("b:build:sn", nil) // last finishes -> span ends + + spans := exp.GetSpans() + var build *tracetest.SpanStub + for i := range spans { + if spans[i].Name == "build" { + build = &spans[i] + } + } + if build == nil { + t.Fatal("no build action span exported") + } + if build.Status.Code != codes.Error { + t.Fatalf("build span status = %v, want Error", build.Status.Code) + } +} From 0b98372285ef0a7f7b39b8691b0a2c46e98651d5 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Mon, 29 Jun 2026 13:24:25 +0200 Subject: [PATCH 26/29] docs: clarify that injected env (incl. feature-gate env) is exported, process env is not --- README.md | 10 +++++++--- docs/config.md | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8e920c3a..e5d457ee 100644 --- a/README.md +++ b/README.md @@ -181,9 +181,13 @@ developer and machine hit a failure, the OS username (`user.name`) and hostname **What is recorded:** the gitte CLI arguments and each action's command line are exported as span attributes (this is intentional — knowing what ran is the -point). Keep secrets out of action command definitions and CLI arguments; pass -them through environment variables, which are **not** exported. Full remote -URLs are never collected either (repos are identified by name only). +point). Note that gitte's **injected** environment — a project's `env`, +`env_when`, and feature-gate env — is also exported on task spans (the +`gitte.env` attribute) and in the task logs; the inherited **process** +environment (everything in `os.Environ`) is **not**. So keep secrets out of +action command definitions, CLI arguments, and config `env`/feature-gate blocks +— pass real secrets through the process environment instead. Full remote URLs +are never collected either (repos are identified by name only). Each `gitte run` produces a structured trace: a root span with child spans for each phase (`startup`, `gitops`, `actions`), a span per startup check, a span diff --git a/docs/config.md b/docs/config.md index 70c2cfd6..72895ef7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -426,7 +426,7 @@ telemetry: Each invocation produces one trace: a root span for the command, child spans for each repo sync (branch, commit SHA, dirty flag) and each action task (command, exit code), with errors recorded on the relevant span. The OS username (`user.name`) and hostname (`host.name`) are attached to every trace to identify which developer and machine produced it. -The gitte CLI arguments and each action's command line are exported as span attributes. Keep secrets out of action command definitions and CLI arguments — pass them via environment variables, which are not exported. Full remote URLs are never collected (repos are identified by name only). +The gitte CLI arguments and each action's command line are exported as span attributes. gitte's injected environment — a project's `env`, `env_when`, and feature-gate env — is also exported (the `gitte.env` span attribute and the task logs); the inherited process environment (`os.Environ`) is not. Keep secrets out of action command definitions, CLI arguments, and config `env`/feature-gate blocks — pass real secrets through the process environment instead. Full remote URLs are never collected (repos are identified by name only). Environment variables override or disable telemetry: From daf77ffc8b5d8e01b9444fdfa5a4e67b36fe1590 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Mon, 29 Jun 2026 13:28:29 +0200 Subject: [PATCH 27/29] feat(telemetry): attach failed task's output tail as gitte.error_tail span attribute --- actions/features_test.go | 21 +++++++++++++++++++++ actions/runner.go | 24 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/actions/features_test.go b/actions/features_test.go index 80fbaa89..93865707 100644 --- a/actions/features_test.go +++ b/actions/features_test.go @@ -2,6 +2,7 @@ package actions import ( "reflect" + "strings" "testing" "github.com/cego/gitte/config" @@ -46,3 +47,23 @@ func TestInjectedEnv(t *testing.T) { t.Fatalf("injectedEnv = %v, want PROJ_VAR=x and FEAT_VAR=1", got) } } + +func TestOutputTail(t *testing.T) { + // stderr preferred when non-empty + if got := outputTail([]byte("the real error"), []byte("noise")); got != "the real error" { + t.Fatalf("stderr-preferred: got %q", got) + } + // falls back to stdout when stderr is blank (e.g. gitlab-ci-local prints to stdout) + if got := outputTail([]byte(" \n"), []byte("stdout failure")); got != "stdout failure" { + t.Fatalf("stdout-fallback: got %q", got) + } + // capped to the last errorTailBytes + big := []byte(strings.Repeat("x", errorTailBytes+500)) + if got := outputTail(big, nil); len(got) != errorTailBytes { + t.Fatalf("cap: len=%d, want %d", len(got), errorTailBytes) + } + // empty when no output + if got := outputTail(nil, nil); got != "" { + t.Fatalf("empty: got %q", got) + } +} diff --git a/actions/runner.go b/actions/runner.go index e543d93e..cbeb61d9 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -347,12 +347,36 @@ func runGroupTask( span.SetAttributes(attribute.Int("gitte.exit_code", res.ExitCode)) if res.ExitCode != 0 { + if tail := outputTail(res.Stderr, res.Stdout); tail != "" { + span.SetAttributes(attribute.String("gitte.error_tail", tail)) + } return fmt.Errorf("command exited with code %d", res.ExitCode) } return nil } +// errorTailBytes caps how much trailing command output is attached to a failed +// task span via the gitte.error_tail attribute. +const errorTailBytes = 4096 + +// outputTail returns the last errorTailBytes of a failed command's output for +// the gitte.error_tail span attribute — stderr preferred, falling back to +// stdout (tools like gitlab-ci-local print their failures to stdout). +func outputTail(stderr, stdout []byte) string { + if tail := tailString(stderr); tail != "" { + return tail + } + return tailString(stdout) +} + +func tailString(b []byte) string { + if len(b) > errorTailBytes { + b = b[len(b)-errorTailBytes:] + } + return strings.TrimSpace(string(b)) +} + // emitTaskPreamble writes a short header to the task log showing the working // directory, command, and any env vars injected by gitte (project env, env_when, // feature gates). It is emitted before the command starts so the log is From 0fa6e11daee12df0ec4cff769132e95705229037 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Mon, 29 Jun 2026 14:23:14 +0200 Subject: [PATCH 28/29] fix: gofmt + keep executor's cancellable ctx for task commands (staticcheck SA4009) --- actions/features_test.go | 4 ++-- actions/runner.go | 6 +++++- telemetry/action_tracker_test.go | 2 +- telemetry/logs_test.go | 2 +- telemetry/telemetry.go | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/actions/features_test.go b/actions/features_test.go index 93865707..ccf981b6 100644 --- a/actions/features_test.go +++ b/actions/features_test.go @@ -12,8 +12,8 @@ import ( func TestEnabledFeaturesForProject(t *testing.T) { cfg := &config.GitteConfig{ FeatureGates: map[string]config.FeatureGate{ - "feat-on": {}, // empty scope → applies to all projects - "feat-off": {}, // disabled in state + "feat-on": {}, // empty scope → applies to all projects + "feat-off": {}, // disabled in state "feat-scoped-out": {Scope: config.FeatureScope{Projects: []string{"other"}}}, // enabled but scoped to a different project }, } diff --git a/actions/runner.go b/actions/runner.go index cbeb61d9..8d347923 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -288,7 +288,11 @@ func runGroupTask( reg *telemetry.SpanRegistry, ) (err error) { actionCtx := tracker.ActionContext(telemetry.ActionOf(taskName)) - ctx, span := telemetry.Tracer().Start(actionCtx, "action.run "+taskName) + // Parent the task span under the action span, but keep running under the + // executor's incoming (cancellable) context so cancellation still propagates + // to the command — attach the span to ctx rather than replacing ctx. + _, span := telemetry.Tracer().Start(actionCtx, "action.run "+taskName) + ctx = trace.ContextWithSpan(ctx, span) reg.Set(taskName, span.SpanContext()) setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) if feats := enabledFeaturesForProject(cfg, st, projName, proj); len(feats) > 0 { diff --git a/telemetry/action_tracker_test.go b/telemetry/action_tracker_test.go index d4658380..e3881f61 100644 --- a/telemetry/action_tracker_test.go +++ b/telemetry/action_tracker_test.go @@ -100,7 +100,7 @@ func TestActionTracker_RecordsTaskErrorOnActionSpan(t *testing.T) { tr.OnStart("a:build:sn") tr.OnStart("b:build:sn") tr.OnFinish("a:build:sn", errors.New("build failed")) // one task fails - tr.OnFinish("b:build:sn", nil) // last finishes -> span ends + tr.OnFinish("b:build:sn", nil) // last finishes -> span ends spans := exp.GetSpans() var build *tracetest.SpanStub diff --git a/telemetry/logs_test.go b/telemetry/logs_test.go index 057687bc..033a4fe8 100644 --- a/telemetry/logs_test.go +++ b/telemetry/logs_test.go @@ -8,8 +8,8 @@ import ( "github.com/cego/gitte/executor" "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/log/global" - "go.opentelemetry.io/otel/trace" sdklog "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/trace" ) func TestSpanRegistry_SetGetDelete(t *testing.T) { diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index dce1a12d..01f40671 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -21,8 +21,8 @@ import ( otlplog "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" otellog "go.opentelemetry.io/otel/log/global" - "go.opentelemetry.io/otel/sdk/resource" sdklog "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" ) From 51227962cd1fb799a5733ba39e23c5f821ba33f1 Mon Sep 17 00:00:00 2001 From: Lau Josefsen Date: Mon, 20 Jul 2026 13:15:16 +0200 Subject: [PATCH 29/29] fix(telemetry): harden export lifecycle and log correlation --- actions/runner.go | 78 ++++++++++++++++++++++++------------- actions/telemetry_test.go | 9 +++++ cmd/root.go | 28 +++++++++++-- cmd/root_telemetry_test.go | 56 ++++++++++++++++++++++++++ config/startup_checks.go | 21 ++++++---- executor/executor.go | 12 +++++- executor/executor_test.go | 21 ++++++++++ executor/types.go | 13 +++++++ startup/startup.go | 39 ++++++++++++++++--- startup/startup_test.go | 56 ++++++++++++++++++++++++++ telemetry/logs.go | 58 ++++----------------------- telemetry/logs_test.go | 71 +++++++++++++++++---------------- telemetry/telemetry.go | 64 +++++++++++++++++++----------- telemetry/telemetry_test.go | 61 ++++++++++++++++++++++++++++- 14 files changed, 432 insertions(+), 155 deletions(-) create mode 100644 cmd/root_telemetry_test.go diff --git a/actions/runner.go b/actions/runner.go index 8d347923..f615e0be 100644 --- a/actions/runner.go +++ b/actions/runner.go @@ -40,7 +40,6 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta view := newView(mode, infos, actionOrder, runCancel, retryCh, cfg.QuickSolve.GitClean.Exclude) tracker := telemetry.NewActionTracker(ctx) - reg := telemetry.NewSpanRegistry() onStart := func(name string) { tracker.OnStart(name) @@ -72,7 +71,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta var retrySet map[string]struct{} // nil on first run var runErr error for { - tasks := buildExecutorTasks(cfg, st, cwd, keys, tracker, reg) + tasks := buildExecutorTasks(cfg, st, cwd, keys, tracker) // Strip needs from explicitly retried tasks so they run immediately. if retrySet != nil { @@ -114,7 +113,7 @@ func RunActions(ctx context.Context, cfg *config.GitteConfig, st *state.GitteSta exec.WithPreCompleted(succeeded, failed) } - exec.WithOutputHandler(telemetry.LogOutputHandler(view.Handler(), reg)) + exec.WithOutputHandler(view.Handler()) exec.WithRetryChannel(retryCh) runErr = exec.Execute(runCtx) @@ -208,7 +207,7 @@ func buildTaskInfos(cfg *config.GitteConfig, st *state.GitteState, cwd string, k } // buildExecutorTasks constructs executor.Task list from keys. -func buildExecutorTasks(cfg *config.GitteConfig, st *state.GitteState, cwd string, keys []GroupKeyWithDeps, tracker *telemetry.ActionTracker, reg *telemetry.SpanRegistry) []executor.Task { +func buildExecutorTasks(cfg *config.GitteConfig, st *state.GitteState, cwd string, keys []GroupKeyWithDeps, tracker *telemetry.ActionTracker) []executor.Task { tasks := make([]executor.Task, 0, len(keys)) searchFors := cfg.SearchFor @@ -253,7 +252,7 @@ func buildExecutorTasks(cfg *config.GitteConfig, st *state.GitteState, cwd strin Needs: needNames, Retry: retryConfig, ExecuteFn: func(ctx context.Context, tName string, handler executor.OutputHandler) error { - return runGroupTask(ctx, cfg, st, cwd, proj, key.Project, tName, cmds, allSearchFors, handler, tracker, reg) + return runGroupTask(ctx, cfg, st, cwd, proj, key.Project, tName, cmds, allSearchFors, handler, tracker) }, }) } @@ -285,7 +284,6 @@ func runGroupTask( searchFors []config.SearchFor, handler executor.OutputHandler, tracker *telemetry.ActionTracker, - reg *telemetry.SpanRegistry, ) (err error) { actionCtx := tracker.ActionContext(telemetry.ActionOf(taskName)) // Parent the task span under the action span, but keep running under the @@ -293,30 +291,21 @@ func runGroupTask( // to the command — attach the span to ctx rather than replacing ctx. _, span := telemetry.Tracer().Start(actionCtx, "action.run "+taskName) ctx = trace.ContextWithSpan(ctx, span) - reg.Set(taskName, span.SpanContext()) - setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) - if feats := enabledFeaturesForProject(cfg, st, projName, proj); len(feats) > 0 { - span.SetAttributes(attribute.StringSlice("gitte.features", feats)) - } - if env := injectedEnv(cfg, st, projName, proj); len(env) > 0 { - keys := make([]string, 0, len(env)) - for k := range env { - keys = append(keys, k) - } - sort.Strings(keys) - kvs := make([]string, 0, len(keys)) - for _, k := range keys { - kvs = append(kvs, k+"="+env[k]) - } - span.SetAttributes(attribute.StringSlice("gitte.env", kvs)) - } + handler = telemetry.LogOutputHandler(handler) + setTaskTelemetryAttrs(span, cfg, st, projName, proj, taskName, cmds) defer func() { + if recovered := recover(); recovered != nil { + panicErr := fmt.Errorf("panic: %v", recovered) + span.RecordError(panicErr) + span.SetStatus(codes.Error, panicErr.Error()) + span.End() + panic(recovered) + } if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) } span.End() - reg.Delete(taskName) }() if len(cmds) == 0 { @@ -348,18 +337,51 @@ func runGroupTask( return err } - span.SetAttributes(attribute.Int("gitte.exit_code", res.ExitCode)) + if span.IsRecording() { + span.SetAttributes(attribute.Int("gitte.exit_code", res.ExitCode)) + if res.ExitCode != 0 { + if tail := outputTail(res.Stderr, res.Stdout); tail != "" { + span.SetAttributes(attribute.String("gitte.error_tail", tail)) + } + } + } if res.ExitCode != 0 { - if tail := outputTail(res.Stderr, res.Stdout); tail != "" { - span.SetAttributes(attribute.String("gitte.error_tail", tail)) - } return fmt.Errorf("command exited with code %d", res.ExitCode) } return nil } +func setTaskTelemetryAttrs( + span trace.Span, + cfg *config.GitteConfig, + st *state.GitteState, + projName string, + proj config.ProjectConfig, + taskName string, + cmds []string, +) { + if span.IsRecording() { + setActionAttrs(span, taskName, projName, strings.Join(cmds, " ")) + if feats := enabledFeaturesForProject(cfg, st, projName, proj); len(feats) > 0 { + span.SetAttributes(attribute.StringSlice("gitte.features", feats)) + } + if env := injectedEnv(cfg, st, projName, proj); len(env) > 0 { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + kvs := make([]string, 0, len(keys)) + for _, k := range keys { + kvs = append(kvs, k+"="+env[k]) + } + span.SetAttributes(attribute.StringSlice("gitte.env", kvs)) + } + } +} + // errorTailBytes caps how much trailing command output is attached to a failed // task span via the gitte.error_tail attribute. const errorTailBytes = 4096 diff --git a/actions/telemetry_test.go b/actions/telemetry_test.go index b38b40fb..19022e79 100644 --- a/actions/telemetry_test.go +++ b/actions/telemetry_test.go @@ -4,9 +4,11 @@ import ( "context" "testing" + "github.com/cego/gitte/config" "go.opentelemetry.io/otel" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace/noop" ) func TestSetActionAttrs(t *testing.T) { @@ -35,3 +37,10 @@ func TestSetActionAttrs(t *testing.T) { t.Fatalf("attrs = %+v", got) } } + +func TestSetTaskTelemetryAttrs_SkipsWorkForNonRecordingSpan(t *testing.T) { + _, span := noop.NewTracerProvider().Tracer("test").Start(context.Background(), "task") + // nil config/state would panic in feature and environment resolution. A + // non-recording span must return before touching either dependency. + setTaskTelemetryAttrs(span, nil, nil, "project", config.ProjectConfig{}, "project:up:default", []string{"true"}) +} diff --git a/cmd/root.go b/cmd/root.go index 108cbce1..1ffa3501 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,7 +36,7 @@ var ( globalCtx context.Context globalCancel context.CancelFunc - globalTelemetryShutdown func() + globalTelemetryShutdown func(context.Context) globalRootSpan trace.Span ) @@ -86,8 +86,7 @@ func Execute() { globalCancel() } }() - err := rootCmd.Execute() - finishTelemetry(err) + err := executeRoot() if err != nil { if output.DetectMode(flagNoTTY) == output.ModePlain { fmt.Fprintln(os.Stderr, "error:", err) @@ -98,6 +97,25 @@ func Execute() { } } +// executeRoot guarantees telemetry finalization for both returned errors and +// panics. A panic is recorded as an error before being re-thrown so callers keep +// the normal panic behavior and stack output. +func executeRoot() (err error) { + return runWithTelemetry(rootCmd.Execute) +} + +func runWithTelemetry(run func() error) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + panicErr := fmt.Errorf("panic: %v", recovered) + finishTelemetry(panicErr) + panic(recovered) + } + finishTelemetry(err) + }() + return run() +} + // finishTelemetry records the final command status on the root span and flushes // pending spans. Safe to call when telemetry was never initialized (e.g. // completion commands or an early config failure), where the handles remain nil. @@ -112,7 +130,9 @@ func finishTelemetry(err error) { globalRootSpan.End() } if globalTelemetryShutdown != nil { - globalTelemetryShutdown() + shutdownCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + globalTelemetryShutdown(shutdownCtx) } } diff --git a/cmd/root_telemetry_test.go b/cmd/root_telemetry_test.go new file mode 100644 index 00000000..ee39618a --- /dev/null +++ b/cmd/root_telemetry_test.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestRunWithTelemetry_RecordsAndFlushesPanicThenRepanics(t *testing.T) { + exporter := tracetest.NewInMemoryExporter() + provider := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + t.Cleanup(func() { _ = provider.Shutdown(context.Background()) }) + + _, span := provider.Tracer("test").Start(context.Background(), "gitte test") + previousSpan := globalRootSpan + previousShutdown := globalTelemetryShutdown + globalRootSpan = span + shutdownCalled := false + globalTelemetryShutdown = func(context.Context) { shutdownCalled = true } + t.Cleanup(func() { + globalRootSpan = previousSpan + globalTelemetryShutdown = previousShutdown + }) + + var recovered any + func() { + defer func() { recovered = recover() }() + _ = runWithTelemetry(func() error { panic("boom") }) + }() + + if recovered != "boom" { + t.Fatalf("recovered panic = %v, want boom", recovered) + } + if !shutdownCalled { + t.Fatal("telemetry shutdown was not called") + } + spans := exporter.GetSpans() + if len(spans) != 1 { + t.Fatalf("exported %d spans, want 1", len(spans)) + } + if spans[0].Status.Code != codes.Error { + t.Fatalf("root span status = %v, want error", spans[0].Status.Code) + } + foundException := false + for _, event := range spans[0].Events { + if event.Name == "exception" { + foundException = true + } + } + if !foundException { + t.Fatal("panic was not sent as an exception event") + } +} diff --git a/config/startup_checks.go b/config/startup_checks.go index 3bb7ec5b..aacd0c02 100644 --- a/config/startup_checks.go +++ b/config/startup_checks.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -18,7 +19,7 @@ type StartupCheck interface { GetType() string GetHint() string GetNeeds() []string - Check(ctx context.Context, cwd string) error + Check(ctx context.Context, cwd string, stdout, stderr io.Writer) error } // BaseStartupCheck holds common fields for all check types @@ -44,14 +45,18 @@ type ShellStartupCheck struct { Script string `yaml:"script"` } -func (s *ShellStartupCheck) Check(ctx context.Context, cwd string) error { +func (s *ShellStartupCheck) Check(ctx context.Context, cwd string, stdout, stderr io.Writer) error { cmd := exec.CommandContext(ctx, s.Shell, "-c", s.Script) //nolint:gosec cmd.Dir = cwd - var stderr bytes.Buffer - cmd.Stderr = &stderr + var stderrBuf bytes.Buffer + if stderr == nil { + stderr = io.Discard + } + cmd.Stdout = stdout + cmd.Stderr = io.MultiWriter(stderr, &stderrBuf) if err := cmd.Run(); err != nil { if exitErr, ok := err.(*exec.ExitError); ok { - stderrStr := strings.TrimSpace(stderr.String()) + stderrStr := strings.TrimSpace(stderrBuf.String()) if stderrStr != "" { return fmt.Errorf("shell script exited with code %d: %s", exitErr.ExitCode(), stderrStr) } @@ -68,12 +73,14 @@ type CommandStartupCheck struct { Command []string `yaml:"cmd"` } -func (s *CommandStartupCheck) Check(ctx context.Context, cwd string) error { +func (s *CommandStartupCheck) Check(ctx context.Context, cwd string, stdout, stderr io.Writer) error { if len(s.Command) == 0 { return fmt.Errorf("command check has no command") } cmd := exec.CommandContext(ctx, s.Command[0], s.Command[1:]...) //nolint:gosec cmd.Dir = cwd + cmd.Stdout = stdout + cmd.Stderr = stderr if err := cmd.Run(); err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Errorf("command exited with code %d", exitErr.ExitCode()) @@ -90,7 +97,7 @@ type YamlPathPresentStartupCheck struct { File string `yaml:"file"` } -func (s *YamlPathPresentStartupCheck) Check(_ context.Context, _ string) error { +func (s *YamlPathPresentStartupCheck) Check(_ context.Context, _ string, _, _ io.Writer) error { path, err := goyaml.PathString(s.Path) if err != nil { return fmt.Errorf("invalid yaml path: %w", err) diff --git a/executor/executor.go b/executor/executor.go index fe2269a0..0d859999 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "runtime/debug" "strconv" "strings" "time" @@ -244,7 +245,7 @@ func (e *Executor) startReadyTasks(ctx context.Context, completionCh chan<- Comm } handler := ToChannelOutputHandler{OutputCh: outputCh} - err := r.task.ExecuteFn(ctx, r.task.Name, handler) + err := executeTask(ctx, r.task, handler) elapsed := time.Since(r.startedAt) if err != nil { @@ -276,6 +277,15 @@ func (e *Executor) startReadyTasks(ctx context.Context, completionCh chan<- Comm return nil } +func executeTask(ctx context.Context, task Task, handler OutputHandler) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = &PanicError{Task: task.Name, Value: recovered, Stack: debug.Stack()} + } + }() + return task.ExecuteFn(ctx, task.Name, handler) +} + // resetForRetry re-queues the named failed tasks and cascades to any skipped dependents. // Returns the number of tasks reset (caller must decrement its finished counter by this amount). func (e *Executor) resetForRetry(names []string) int { diff --git a/executor/executor_test.go b/executor/executor_test.go index 9c405592..8823fbb7 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -152,6 +152,27 @@ func TestExecutor_TaskFailureSkipsDependents(t *testing.T) { } } +func TestExecutor_TaskPanicBecomesError(t *testing.T) { + tasks := []Task{{ + Name: "panicking", + ExecuteFn: func(context.Context, string, OutputHandler) error { + panic("boom") + }, + }} + exec, err := NewExecutor(tasks, ExecutorOptions{}) + if err != nil { + t.Fatalf("NewExecutor() error = %v", err) + } + err = exec.Execute(context.Background()) + var panicErr *PanicError + if !errors.As(err, &panicErr) { + t.Fatalf("Execute() error = %v, want PanicError", err) + } + if panicErr.Value != "boom" || len(panicErr.Stack) == 0 { + t.Fatalf("PanicError = %+v", panicErr) + } +} + func TestExecutor_SkippedErrorWrapsErrTaskSkipped(t *testing.T) { var skippedErr error tasks := []Task{ diff --git a/executor/types.go b/executor/types.go index 5e6560c3..0908b53c 100644 --- a/executor/types.go +++ b/executor/types.go @@ -2,6 +2,7 @@ package executor import ( "context" + "fmt" "time" ) @@ -83,3 +84,15 @@ type CommandResult struct { Success bool Error error } + +// PanicError converts a task-worker panic into an error so executor output and +// telemetry can drain before the failure reaches the command root. +type PanicError struct { + Task string + Value any + Stack []byte +} + +func (e *PanicError) Error() string { + return fmt.Sprintf("task %s panicked: %v\n%s", e.Task, e.Value, e.Stack) +} diff --git a/startup/startup.go b/startup/startup.go index 064a2b76..f2a7e790 100644 --- a/startup/startup.go +++ b/startup/startup.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "sort" "github.com/cego/gitte/config" @@ -24,7 +25,6 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O ctx, cancel := context.WithCancel(ctx) defer cancel() - reg := telemetry.NewSpanRegistry() tasks := make([]executor.Task, 0, len(cfg.StartupChecks)) for name, check := range cfg.StartupChecks { name := name @@ -34,16 +34,24 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O Needs: check.GetNeeds(), ExecuteFn: func(ctx context.Context, taskName string, handler executor.OutputHandler) (err error) { ctx, span := startCheckSpan(ctx, taskName) - reg.Set(taskName, span.SpanContext()) defer func() { + if recovered := recover(); recovered != nil { + panicErr := fmt.Errorf("panic: %v", recovered) + span.RecordError(panicErr) + span.SetStatus(codes.Error, panicErr.Error()) + span.End() + panic(recovered) + } if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) } span.End() - reg.Delete(taskName) }() - if cerr := check.Check(ctx, cwd); cerr != nil { + logHandler := telemetry.LogOutputHandler(handler) + stdout := &handlerWriter{ctx: ctx, handler: logHandler, taskName: taskName, stream: executor.StdoutStream} + stderr := &handlerWriter{ctx: ctx, handler: logHandler, taskName: taskName, stream: executor.StderrStream} + if cerr := check.Check(ctx, cwd, stdout, stderr); cerr != nil { hint := check.GetHint() if hint != "" { return fmt.Errorf("%s\nhint: %s", cerr.Error(), hint) @@ -65,8 +73,6 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O if err != nil { return fmt.Errorf("startup checks have invalid dependencies: %w", err) } - exec.WithOutputHandler(telemetry.LogOutputHandler(executor.NoopOutputHandler{}, reg)) - runErr := exec.Execute(ctx) view.Wait() if runErr != nil && mode != output.ModePlain { @@ -77,6 +83,27 @@ func Run(ctx context.Context, cfg *config.GitteConfig, cwd string, mode output.O return runErr } +// handlerWriter adapts startup checks using io.Writer to executor output while +// retaining the check span in the context used for correlated OTEL logs. +type handlerWriter struct { + ctx context.Context + handler executor.OutputHandler + taskName string + stream executor.StreamType +} + +var _ io.Writer = (*handlerWriter)(nil) + +func (w *handlerWriter) Write(p []byte) (int, error) { + line := append([]byte(nil), p...) + _ = w.handler.HandleOutput(w.ctx, executor.Output{ + Output: line, + CmdName: w.taskName, + Stream: w.stream, + }) + return len(p), nil +} + // startCheckSpan opens a span for a single startup check. func startCheckSpan(ctx context.Context, name string) (context.Context, trace.Span) { return telemetry.Tracer().Start(ctx, "startup.check "+name) diff --git a/startup/startup_test.go b/startup/startup_test.go index 1bab0bbf..1882d134 100644 --- a/startup/startup_test.go +++ b/startup/startup_test.go @@ -2,9 +2,14 @@ package startup import ( "context" + "sync" "testing" + "github.com/cego/gitte/config" + "github.com/cego/gitte/output" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/log/global" + sdklog "go.opentelemetry.io/otel/sdk/log" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" ) @@ -25,3 +30,54 @@ func TestStartCheckSpan_RecordsNamedSpan(t *testing.T) { t.Fatalf("got %+v, want one span named 'startup.check git-present'", spans) } } + +type startupLogExporter struct { + mu sync.Mutex + records []sdklog.Record +} + +func (e *startupLogExporter) Export(_ context.Context, records []sdklog.Record) error { + e.mu.Lock() + defer e.mu.Unlock() + for _, record := range records { + e.records = append(e.records, record.Clone()) + } + return nil +} + +func (*startupLogExporter) Shutdown(context.Context) error { return nil } +func (*startupLogExporter) ForceFlush(context.Context) error { return nil } + +func TestRun_ExportsStartupCommandOutputWithSpanCorrelation(t *testing.T) { + logExp := &startupLogExporter{} + lp := sdklog.NewLoggerProvider(sdklog.WithProcessor(sdklog.NewSimpleProcessor(logExp))) + prevLP := global.GetLoggerProvider() + global.SetLoggerProvider(lp) + t.Cleanup(func() { global.SetLoggerProvider(prevLP); _ = lp.Shutdown(context.Background()) }) + + spanExp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(spanExp)) + prevTP := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { otel.SetTracerProvider(prevTP); _ = tp.Shutdown(context.Background()) }) + + cfg := &config.GitteConfig{StartupChecks: config.StartupCheckMap{ + "output-check": &config.ShellStartupCheck{ + BaseStartupCheck: config.BaseStartupCheck{Type: "shell"}, + Shell: "sh", + Script: "printf stdout-line; printf stderr-line >&2", + }, + }} + if err := Run(context.Background(), cfg, t.TempDir(), output.ModePlain); err != nil { + t.Fatalf("Run() error = %v", err) + } + + if len(logExp.records) != 2 { + t.Fatalf("exported %d startup log records, want 2", len(logExp.records)) + } + for _, record := range logExp.records { + if !record.TraceID().IsValid() || !record.SpanID().IsValid() { + t.Fatalf("startup log is not span-correlated: trace=%s span=%s", record.TraceID(), record.SpanID()) + } + } +} diff --git a/telemetry/logs.go b/telemetry/logs.go index ad1efdb1..8269f5d7 100644 --- a/telemetry/logs.go +++ b/telemetry/logs.go @@ -8,65 +8,26 @@ import ( "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/log/global" - "go.opentelemetry.io/otel/trace" ) -// SpanRegistry maps a task name to the SpanContext of the span representing that -// task, so output lines (which arrive on a separate goroutine without the task -// span in their context) can be correlated to the right span. -type SpanRegistry struct { - mu sync.RWMutex - m map[string]trace.SpanContext -} - -// NewSpanRegistry returns an empty, concurrency-safe registry. -func NewSpanRegistry() *SpanRegistry { - return &SpanRegistry{m: make(map[string]trace.SpanContext)} -} - -// Set stores the SpanContext sc for task, overwriting any existing entry. -func (r *SpanRegistry) Set(task string, sc trace.SpanContext) { - r.mu.Lock() - r.m[task] = sc - r.mu.Unlock() -} - -// Get returns the SpanContext stored for task and whether it was found. -func (r *SpanRegistry) Get(task string) (trace.SpanContext, bool) { - r.mu.RLock() - sc, ok := r.m[task] - r.mu.RUnlock() - return sc, ok -} - -// Delete removes the entry for task from the registry. -func (r *SpanRegistry) Delete(task string) { - r.mu.Lock() - delete(r.m, task) - r.mu.Unlock() -} - // logHandler forwards output to inner and emits a correlated OTEL log record. type logHandler struct { inner executor.OutputHandler - reg *SpanRegistry once sync.Once lgr log.Logger } // LogOutputHandler wraps inner so that every output line is also emitted as an -// OTEL log record correlated (via reg) to the span for output.CmdName. When -// logs are disabled the global logger provider is a no-op, so this is safe and -// cheap; output is always forwarded to inner unchanged. +// OTEL log record correlated to the span in ctx. Callers should install this +// wrapper at the producer boundary, before output is handed to an asynchronous +// drain, so the task span is still available. When logs are disabled the global +// logger provider is a no-op; output is always forwarded unchanged. // // The logger is resolved lazily on first use so that callers constructed before // telemetry.Init registers the real LoggerProvider still pick up the live // provider. -func LogOutputHandler(inner executor.OutputHandler, reg *SpanRegistry) executor.OutputHandler { - return &logHandler{ - inner: inner, - reg: reg, - } +func LogOutputHandler(inner executor.OutputHandler) executor.OutputHandler { + return &logHandler{inner: inner} } func (h *logHandler) logger() log.Logger { @@ -96,10 +57,5 @@ func (h *logHandler) emit(ctx context.Context, out executor.Output) { if len(out.Output) >= 6 && string(out.Output[:6]) == "[HINT]" { rec.AddAttributes(log.Bool("gitte.hint", true)) } - // Correlate to the task span if we know it. - emitCtx := ctx - if sc, ok := h.reg.Get(out.CmdName); ok && sc.IsValid() { - emitCtx = trace.ContextWithSpanContext(ctx, sc) - } - h.logger().Emit(emitCtx, rec) + h.logger().Emit(ctx, rec) } diff --git a/telemetry/logs_test.go b/telemetry/logs_test.go index 033a4fe8..bc6d09ec 100644 --- a/telemetry/logs_test.go +++ b/telemetry/logs_test.go @@ -9,26 +9,10 @@ import ( "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/log/global" sdklog "go.opentelemetry.io/otel/sdk/log" - "go.opentelemetry.io/otel/trace" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" ) -func TestSpanRegistry_SetGetDelete(t *testing.T) { - reg := NewSpanRegistry() - sc := trace.NewSpanContext(trace.SpanContextConfig{ - TraceID: trace.TraceID{1}, - SpanID: trace.SpanID{2}, - }) - reg.Set("proj:build:sn", sc) - got, ok := reg.Get("proj:build:sn") - if !ok || got.TraceID() != sc.TraceID() { - t.Fatalf("Get = %v, %v; want %v", got, ok, sc) - } - reg.Delete("proj:build:sn") - if _, ok := reg.Get("proj:build:sn"); ok { - t.Fatal("expected entry removed after Delete") - } -} - type recordingHandler struct{ lines []string } func (r *recordingHandler) HandleOutput(_ context.Context, o executor.Output) error { @@ -40,8 +24,7 @@ func TestLogOutputHandler_ForwardsUnchanged(t *testing.T) { // With logs disabled (no provider), the wrapper must still forward output // to the inner handler and never error. inner := &recordingHandler{} - reg := NewSpanRegistry() - h := LogOutputHandler(inner, reg) + h := LogOutputHandler(inner) err := h.HandleOutput(context.Background(), executor.Output{ Output: []byte("hello"), CmdName: "proj:build:sn", Stream: executor.StdoutStream, }) @@ -99,8 +82,7 @@ func TestLogOutputHandler_StdoutSeverityInfo(t *testing.T) { exp := setupRecordingProvider(t) inner := &recordingHandler{} - reg := NewSpanRegistry() - h := LogOutputHandler(inner, reg) + h := LogOutputHandler(inner) _ = h.HandleOutput(context.Background(), executor.Output{ Output: []byte("normal line"), @@ -121,8 +103,7 @@ func TestLogOutputHandler_StderrSeverityWarn(t *testing.T) { exp := setupRecordingProvider(t) inner := &recordingHandler{} - reg := NewSpanRegistry() - h := LogOutputHandler(inner, reg) + h := LogOutputHandler(inner) _ = h.HandleOutput(context.Background(), executor.Output{ Output: []byte("error output"), @@ -143,8 +124,7 @@ func TestLogOutputHandler_HintAttribute(t *testing.T) { exp := setupRecordingProvider(t) inner := &recordingHandler{} - reg := NewSpanRegistry() - h := LogOutputHandler(inner, reg) + h := LogOutputHandler(inner) _ = h.HandleOutput(context.Background(), executor.Output{ Output: []byte("[HINT] do something"), @@ -175,16 +155,39 @@ func TestLogOutputHandler_HintAttribute(t *testing.T) { } } -func TestLogsEndpointURL(t *testing.T) { - cases := []struct{ in, want string }{ - {"https://apm.example.com", "https://apm.example.com/v1/logs"}, - {"https://apm.example.com/", "https://apm.example.com/v1/logs"}, - {"https://apm.example.com:8200", "https://apm.example.com:8200/v1/logs"}, - {"https://apm.example.com/custom/logs", "https://apm.example.com/custom/logs"}, +func TestLogOutputHandler_UsesProducerSpanContext(t *testing.T) { + exp := setupRecordingProvider(t) + spanExp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(spanExp)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + + ctx, span := tp.Tracer("test").Start(context.Background(), "task") + wantTraceID := span.SpanContext().TraceID() + wantSpanID := span.SpanContext().SpanID() + h := LogOutputHandler(&recordingHandler{}) + _ = h.HandleOutput(ctx, executor.Output{Output: []byte("last line"), CmdName: "task", Stream: executor.StderrStream}) + span.End() + + recs := exp.Records() + if len(recs) != 1 { + t.Fatalf("expected 1 record, got %d", len(recs)) + } + if recs[0].TraceID() != wantTraceID || recs[0].SpanID() != wantSpanID { + t.Fatalf("log correlation = %s/%s, want %s/%s", recs[0].TraceID(), recs[0].SpanID(), wantTraceID, wantSpanID) + } +} + +func TestSignalEndpointURL(t *testing.T) { + cases := []struct{ endpoint, signal, want string }{ + {"https://apm.example.com", "traces", "https://apm.example.com/v1/traces"}, + {"https://apm.example.com/", "traces", "https://apm.example.com/v1/traces"}, + {"https://apm.example.com:8200/", "logs", "https://apm.example.com:8200/v1/logs"}, + {"https://apm.example.com/custom/traces", "traces", "https://apm.example.com/custom/traces"}, + {"https://apm.example.com/?token=x", "traces", "https://apm.example.com/v1/traces?token=x"}, } for _, c := range cases { - if got := logsEndpointURL(c.in); got != c.want { - t.Errorf("logsEndpointURL(%q) = %q, want %q", c.in, got, c.want) + if got := signalEndpointURL(c.endpoint, c.signal); got != c.want { + t.Errorf("signalEndpointURL(%q, %q) = %q, want %q", c.endpoint, c.signal, got, c.want) } } } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 01f40671..35a36367 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -12,6 +12,7 @@ import ( "os/user" "runtime" "strings" + "sync" "time" "github.com/cego/gitte/config" @@ -29,7 +30,7 @@ import ( const tracerName = "github.com/cego/gitte" -// flushTimeout bounds how long exit can block flushing spans. Kept short so an +// flushTimeout bounds how long exit can block flushing each signal. Kept short so an // enabled-but-unreachable endpoint (e.g. laptop with the VPN off) adds at most // this delay to every command. const flushTimeout = 1 * time.Second @@ -114,10 +115,10 @@ func resourceAttributes(version, username, hostname string) []attribute.KeyValue // that flushes pending spans with a bounded timeout. The returned function is // always non-nil and safe to call; setup failures and disabled telemetry both // degrade to a no-op shutdown. -func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { +func Init(ctx context.Context, cfg *config.GitteConfig, version string) func(context.Context) { r := Resolve(cfg) if !r.Enabled { - return func() {} + return func(context.Context) {} } // Only mutate process-wide OTEL state once telemetry is known to be enabled. @@ -130,7 +131,7 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { var opts []otlptracehttp.Option if !r.UseSDKEnv { - opts = append(opts, otlptracehttp.WithEndpointURL(r.Endpoint)) + opts = append(opts, otlptracehttp.WithEndpointURL(signalEndpointURL(r.Endpoint, "traces"))) if len(r.Headers) > 0 { opts = append(opts, otlptracehttp.WithHeaders(r.Headers)) } @@ -139,7 +140,7 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { exporter, err := otlptracehttp.New(ctx, opts...) if err != nil { // Never block gitte: disable telemetry on exporter setup failure. - return func() {} + return func(context.Context) {} } username := "" @@ -152,7 +153,6 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { tp := sdktrace.NewTracerProvider( sdktrace.WithBatcher(exporter), sdktrace.WithResource(res), - sdktrace.WithSampler(sdktrace.AlwaysSample()), ) otel.SetTracerProvider(tp) @@ -160,7 +160,7 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { if logsEnabled() { var logOpts []otlplog.Option if !r.UseSDKEnv { - logOpts = append(logOpts, otlplog.WithEndpointURL(logsEndpointURL(r.Endpoint))) + logOpts = append(logOpts, otlplog.WithEndpointURL(signalEndpointURL(r.Endpoint, "logs"))) if len(r.Headers) > 0 { logOpts = append(logOpts, otlplog.WithHeaders(r.Headers)) } @@ -174,34 +174,54 @@ func Init(ctx context.Context, cfg *config.GitteConfig, version string) func() { } } - return func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), flushTimeout) - defer cancel() - _ = tp.Shutdown(shutdownCtx) + return func(ctx context.Context) { + providers := []shutdowner{tp} if lp != nil { - _ = lp.Shutdown(shutdownCtx) + providers = append(providers, lp) } + shutdownProviders(ctx, providers...) } } +type shutdowner interface { + Shutdown(context.Context) error +} + +// shutdownProviders flushes signals concurrently. Each provider receives its +// own timeout so one slow exporter cannot consume another signal's budget. +func shutdownProviders(ctx context.Context, providers ...shutdowner) { + var wg sync.WaitGroup + for _, provider := range providers { + provider := provider + wg.Add(1) + go func() { + defer wg.Done() + shutdownCtx, cancel := context.WithTimeout(ctx, flushTimeout) + defer cancel() + _ = provider.Shutdown(shutdownCtx) + }() + } + wg.Wait() +} + // logsEnabled reports whether OTEL logs should be exported (enabled with tracing // unless GITTE_TELEMETRY_LOGS=off). func logsEnabled() bool { return !strings.EqualFold(os.Getenv("GITTE_TELEMETRY_LOGS"), "off") } -// logsEndpointURL returns the OTLP/HTTP logs endpoint for a configured base -// endpoint. otlploghttp.WithEndpointURL uses the URL's path verbatim, so a -// path-less endpoint (e.g. "https://apm.example.com") would POST to the server -// root and be rejected. When the endpoint has no path we append the standard -// "/v1/logs" intake path (matching how the traces exporter targets -// "/v1/traces"); an endpoint that already carries a path is left untouched. -func logsEndpointURL(endpoint string) string { +// signalEndpointURL returns an OTLP/HTTP endpoint for signal. EndpointURL +// options use a URL path verbatim, so path-less and root-path configured URLs +// need the standard signal intake path appended. Custom paths are preserved. +func signalEndpointURL(endpoint, signal string) string { u, err := url.Parse(endpoint) - if err != nil || u.Path == "" || u.Path == "/" { - return strings.TrimRight(endpoint, "/") + "/v1/logs" + if err != nil { + return endpoint + } + if u.Path == "" || u.Path == "/" { + u.Path = "/v1/" + signal } - return endpoint + return u.String() } // Tracer returns gitte's tracer from the global provider (a no-op tracer when diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 12f6e4fa..e8645a45 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -2,7 +2,9 @@ package telemetry import ( "context" + "sync" "testing" + "time" "github.com/cego/gitte/config" "go.opentelemetry.io/otel" @@ -108,7 +110,7 @@ func TestInit_DisabledReturnsNoopShutdown(t *testing.T) { if shutdown == nil { t.Fatal("shutdown must never be nil") } - shutdown() // must not panic + shutdown(context.Background()) // must not panic } func TestInit_EnabledReturnsCallableShutdown(t *testing.T) { @@ -127,7 +129,62 @@ func TestInit_EnabledReturnsCallableShutdown(t *testing.T) { if shutdown == nil { t.Fatal("shutdown must never be nil on the enabled path") } - shutdown() // must not panic or hang beyond the flush timeout + shutdown(context.Background()) // must not panic or hang beyond the flush timeout +} + +func TestInit_RespectsOTELTracesSampler(t *testing.T) { + t.Setenv("GITTE_TELEMETRY", "") + t.Setenv("GITTE_TELEMETRY_LOGS", "off") + t.Setenv("OTEL_TRACES_SAMPLER", "always_off") + prev := otel.GetTracerProvider() + t.Cleanup(func() { otel.SetTracerProvider(prev) }) + + shutdown := Init(context.Background(), &config.GitteConfig{Telemetry: config.TelemetryConfig{Endpoint: "http://localhost:4318"}}, "test") + _, span := Tracer().Start(context.Background(), "not-recorded") + if span.IsRecording() { + t.Fatal("span is recording despite OTEL_TRACES_SAMPLER=always_off") + } + span.End() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + shutdown(ctx) +} + +type testShutdowner struct { + called chan struct{} + wait bool + once sync.Once +} + +func (s *testShutdowner) Shutdown(ctx context.Context) error { + s.once.Do(func() { close(s.called) }) + if s.wait { + <-ctx.Done() + } + return nil +} + +func TestShutdownProviders_RunIndependentlyAndHonorCancellation(t *testing.T) { + traceProvider := &testShutdowner{called: make(chan struct{}), wait: true} + logProvider := &testShutdowner{called: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + shutdownProviders(ctx, traceProvider, logProvider) + close(done) + }() + + select { + case <-logProvider.called: + case <-time.After(100 * time.Millisecond): + t.Fatal("log shutdown was blocked behind trace shutdown") + } + cancel() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Fatal("shutdown did not stop after cancellation") + } } func TestStartCommandSpan_NoProviderDoesNotPanic(t *testing.T) {