From 1e310e9b446871cda5a3275bc0680f369e51dd1a Mon Sep 17 00:00:00 2001 From: Vijay Date: Tue, 13 Oct 2026 18:53:11 -0400 Subject: [PATCH 1/3] refactor: share bounded import accounting and attribution helpers Signed-off-by: Vijay --- internal/accounting/cost.go | 32 + internal/attribution/hash.go | 56 ++ internal/attribution/hash_test.go | 38 + internal/boundedjson/decode.go | 88 +++ internal/cli/refactor_golden_test.go | 54 ++ .../cli/testdata/import-refactor/README.md | 14 + .../import-refactor/cache-periods.json | 452 +++++++++++ .../import-refactor/cache-periods.text | 32 + .../cli/testdata/import-refactor/inspect.json | 358 +++++++++ .../cli/testdata/import-refactor/inspect.text | 54 ++ .../import-refactor/session-review.json | 261 ++++++ .../import-refactor/session-review.text | 36 + .../import-refactor/spend-periods.json | 748 ++++++++++++++++++ .../import-refactor/spend-periods.text | 42 + internal/inspect/decode.go | 65 +- internal/spend/accounting.go | 19 +- internal/spend/identity_golden_test.go | 58 ++ internal/spend/investigate.go | 35 +- internal/spend/testdata/legacy-identity.json | 39 + 19 files changed, 2370 insertions(+), 111 deletions(-) create mode 100644 internal/accounting/cost.go create mode 100644 internal/attribution/hash.go create mode 100644 internal/attribution/hash_test.go create mode 100644 internal/boundedjson/decode.go create mode 100644 internal/cli/refactor_golden_test.go create mode 100644 internal/cli/testdata/import-refactor/README.md create mode 100644 internal/cli/testdata/import-refactor/cache-periods.json create mode 100644 internal/cli/testdata/import-refactor/cache-periods.text create mode 100644 internal/cli/testdata/import-refactor/inspect.json create mode 100644 internal/cli/testdata/import-refactor/inspect.text create mode 100644 internal/cli/testdata/import-refactor/session-review.json create mode 100644 internal/cli/testdata/import-refactor/session-review.text create mode 100644 internal/cli/testdata/import-refactor/spend-periods.json create mode 100644 internal/cli/testdata/import-refactor/spend-periods.text create mode 100644 internal/spend/identity_golden_test.go create mode 100644 internal/spend/testdata/legacy-identity.json diff --git a/internal/accounting/cost.go b/internal/accounting/cost.go new file mode 100644 index 0000000..61f9ffd --- /dev/null +++ b/internal/accounting/cost.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package accounting + +import ( + "math/big" + "regexp" +) + +var decimalPattern = regexp.MustCompile(`^[0-9]{1,19}(\.[0-9]{1,20})?([eE][+-]?[0-9]{1,3})?$`) +var maxCost = new(big.Rat).SetInt(new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)) + +// Cost preserves the exact bounded source decimal, not provider billing truth. +func Cost(s string) (*big.Rat, bool) { + if len(s) > 48 || !decimalPattern.MatchString(s) { + return nil, false + } + r, ok := new(big.Rat).SetString(s) + if !ok || r.Sign() < 0 || r.Cmp(maxCost) > 0 { + return nil, false + } + if digits, exact := r.FloatPrec(); !exact || digits > 344 { + return nil, false + } + return r, true +} + +func CostText(n *big.Rat) string { + digits, _ := n.FloatPrec() + return n.FloatString(digits) +} diff --git a/internal/attribution/hash.go b/internal/attribution/hash.go new file mode 100644 index 0000000..515ad24 --- /dev/null +++ b/internal/attribution/hash.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +// Package attribution shares local key handling and source-separated identities. +package attribution + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/json" + "errors" + "os" + + "github.com/llm-measurement/llm-sketchkit/go/sketchkit/canon" + sketchhash "github.com/llm-measurement/llm-sketchkit/go/sketchkit/hash" +) + +func Secret(env string) ([]byte, string, error) { + if env == "" { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return nil, "", errors.New("cannot generate a local hashing secret") + } + return b, "ephemeral", nil + } + if len(env) > 128 || len(os.Getenv(env)) > 4096 { + return nil, "", errors.New("invalid hash secret environment setting") + } + if _, err := sketchhash.SecretFromEnv(env); err != nil { + return nil, "", errors.New("hash secret missing or weak; use a random secret of at least 16 bytes") + } + return []byte(os.Getenv(env)), "environment", nil +} + +// Identity preserves the established source/kind/component framing and text_v1. +func Identity(key []byte, source, kind string, parts ...string) ([32]byte, error) { + canonical := []string{source, kind} + for _, part := range parts { + b, err := canon.CanonicalizeString(canon.TextV1, part) + if err != nil { + return [32]byte{}, errors.New("identity is not valid UTF-8") + } + canonical = append(canonical, string(b)) + } + b, _ := json.Marshal(canonical) + return Digest(key, string(sketchhash.UserV1), b), nil +} + +func Digest(key []byte, domain string, data []byte) [32]byte { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(domain)) + mac.Write([]byte{0}) + mac.Write(data) + return [32]byte(mac.Sum(nil)) +} diff --git a/internal/attribution/hash_test.go b/internal/attribution/hash_test.go new file mode 100644 index 0000000..f584514 --- /dev/null +++ b/internal/attribution/hash_test.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package attribution + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/json" + "testing" + + "github.com/llm-measurement/llm-sketchkit/go/sketchkit/canon" +) + +func TestExistingSpendIdentityBytesPreserved(t *testing.T) { + key := []byte("synthetic-regression-key-32-bytes-long") + for _, parts := range [][]string{{" Key A ", "S\r\nession"}, {"caf\u0065\u0301"}, {"\u001c"}, {""}} { + legacy := []string{"litellm-spend/v1", "session"} + for _, p := range parts { + b, e := canon.CanonicalizeString(canon.TextV1, p) + if e != nil { + t.Fatal(e) + } + legacy = append(legacy, string(b)) + } + b, _ := json.Marshal(legacy) + mac := hmac.New(sha256.New, key) + mac.Write([]byte("user:v1\x00")) + mac.Write(b) + got, e := Identity(key, "litellm-spend/v1", "session", parts...) + if e != nil { + t.Fatal(e) + } + if got != [32]byte(mac.Sum(nil)) { + t.Fatalf("existing spend identity changed: %x != %x", got, mac.Sum(nil)) + } + } +} diff --git a/internal/boundedjson/decode.go b/internal/boundedjson/decode.go new file mode 100644 index 0000000..1225811 --- /dev/null +++ b/internal/boundedjson/decode.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +// Package boundedjson decodes bounded JSON without accepting duplicate members. +package boundedjson + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "unicode/utf8" +) + +var errEncoding = errors.New("invalid or over-limit JSON structure") + +// Decode composes encoding/json with the inspect decoder's structural limits. +// Callers must bound bytes before calling; Number preserves decimal lexemes. +func Decode(data []byte, maxDepth, maxNodes int) (any, error) { + if !utf8.Valid(data) { + return nil, errEncoding + } + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + value, err := read(d, 0, maxDepth, &maxNodes) + if err != nil { + return nil, errEncoding + } + if _, err := d.Token(); !errors.Is(err, io.EOF) { + return nil, errEncoding + } + return value, nil +} + +func read(d *json.Decoder, depth, maxDepth int, left *int) (any, error) { + *left-- + if depth > maxDepth || *left < 0 { + return nil, errEncoding + } + t, err := d.Token() + if err != nil { + return nil, err + } + switch t { + case json.Delim('{'): + m := map[string]any{} + for d.More() { + t, err := d.Token() + if err != nil { + return nil, err + } + key, ok := t.(string) + if !ok { + return nil, errEncoding + } + if _, duplicate := m[key]; duplicate { + return nil, errEncoding + } + v, err := read(d, depth+1, maxDepth, left) + if err != nil { + return nil, err + } + m[key] = v + } + if t, err := d.Token(); err != nil || t != json.Delim('}') { + return nil, errEncoding + } + return m, nil + case json.Delim('['): + a := []any{} + for d.More() { + v, err := read(d, depth+1, maxDepth, left) + if err != nil { + return nil, err + } + a = append(a, v) + } + if t, err := d.Token(); err != nil || t != json.Delim(']') { + return nil, errEncoding + } + return a, nil + default: + if _, bad := t.(json.Delim); bad { + return nil, errEncoding + } + return t, nil + } +} diff --git a/internal/cli/refactor_golden_test.go b/internal/cli/refactor_golden_test.go new file mode 100644 index 0000000..a7e75a2 --- /dev/null +++ b/internal/cli/refactor_golden_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +// These bytes were recorded from v0.8.0, before extracting shared import helpers. +func TestImportRefactorGoldenOutputs(t *testing.T) { + t.Setenv("FLEETDIFF_GOLDEN_KEY", "synthetic-refactor-regression-key-32-bytes") + for _, tc := range []struct { + name string + args []string + }{ + {"inspect", []string{"inspect", "--secret-env", "FLEETDIFF_GOLDEN_KEY", "--show-hashes"}}, + {"spend-periods", []string{"investigate", "--litellm-spend", "../../examples/litellm-spend/synthetic.csv", "--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "session", "--hash-secret-env", "FLEETDIFF_GOLDEN_KEY", "--show-hashes"}}, + {"session-review", []string{"investigate", "--litellm-spend", "../../examples/litellm-spend/session-budget.csv", "--period", "2026-10-03", "--session-budget-usd", "1", "--hash-secret-env", "FLEETDIFF_GOLDEN_KEY", "--show-hashes"}}, + {"cache-periods", []string{"investigate", "--litellm-spend", "../../examples/litellm-spend/cache-drop.csv", "--before-period", "2026-10-07", "--after-period", "2026-10-08", "--hash-secret-env", "FLEETDIFF_GOLDEN_KEY", "--show-hashes"}}, + } { + for _, format := range []string{"text", "json"} { + t.Run(tc.name+"/"+format, func(t *testing.T) { + args := append(append([]string{}, tc.args...), "--format", format) + if tc.name == "inspect" { + args = append(args, "../../examples/inspect/synthetic-traces.json") + } + var out, errout bytes.Buffer + if code := Run(args, &out, &errout); code != 0 || errout.Len() != 0 { + t.Fatalf("exit=%d stderr=%s", code, &errout) + } + path := filepath.Join("testdata", "import-refactor", tc.name+"."+format) + if os.Getenv("FLEETDIFF_UPDATE_REFACTOR_GOLDENS") == "1" { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, out.Bytes(), 0644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(out.Bytes(), want) { + t.Fatalf("output differs from v0.8.0 golden %s (got %d bytes, want %d)", path, out.Len(), len(want)) + } + }) + } + } +} diff --git a/internal/cli/testdata/import-refactor/README.md b/internal/cli/testdata/import-refactor/README.md new file mode 100644 index 0000000..4a4ccd8 --- /dev/null +++ b/internal/cli/testdata/import-refactor/README.md @@ -0,0 +1,14 @@ +# Import Helper Refactor Goldens + +These text and JSON outputs were recorded from unchanged fleetdiff v0.8.0 +(`44967bd743a6fa26588742f67c54df6e323e90e0`) before extracting the shared helpers. +They cover inspect, two-period spend, cache comparison and session-budget review. +The synthetic inputs already live in `examples/`; hashes use a fixed public test +key. `internal/spend/testdata/legacy-identity.json` also pins legacy hash bytes, +including Unicode and empty-identity cases. + +The tests compare complete output bytes, without deleting or normalizing fields. +The same tests must pass before and after an extraction. To deliberately update +these records, use `FLEETDIFF_UPDATE_REFACTOR_GOLDENS=1` with the two named tests, +review the resulting bytes and document the baseline change here. Ordinary CI +runs without that variable. diff --git a/internal/cli/testdata/import-refactor/cache-periods.json b/internal/cli/testdata/import-refactor/cache-periods.json new file mode 100644 index 0000000..4774b70 --- /dev/null +++ b/internal/cli/testdata/import-refactor/cache-periods.json @@ -0,0 +1,452 @@ +{ + "cache": { + "before": { + "requests": 2, + "read": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 1840, + "share_of_recorded_input": 0.92 + }, + "write": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 100, + "share_of_recorded_input": 0.05 + }, + "uncached_input_tokens": 160, + "read_covered_requests_without_group_identity": 0 + }, + "after": { + "requests": 2, + "read": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 1220, + "share_of_recorded_input": 0.61 + }, + "write": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 150, + "share_of_recorded_input": 0.075 + }, + "uncached_input_tokens": 780, + "read_covered_requests_without_group_identity": 0 + }, + "uncached_growth_rankings": { + "name": "uncached_key", + "weight_unit": "attributed-recorded-uncached-input-tokens", + "before_weight": 160, + "after_weight": 780, + "before_max_error": 0, + "after_max_error": 0, + "candidate_count": 2, + "untracked_delta_bound": { + "lower": 0, + "upper": 0 + }, + "tracked_movers": [ + { + "item": "key-1", + "hash": "977a7cf5669d8e11", + "before": { + "lower": 80, + "upper": 80 + }, + "after": { + "lower": 700, + "upper": 700 + }, + "delta": { + "lower": 620, + "upper": 620 + }, + "direction": "increased" + } + ] + } + }, + "comparison_incomplete": false, + "call_types": [ + { + "call_type": "completion", + "analyzed": true, + "before": { + "cache_read_detail_recorded_requests": 2, + "cache_write_detail_recorded_requests": 2, + "requests": 2, + "invalid_usage": 0, + "missing_usage": 0, + "prompt_tokens": 2000, + "completion_tokens": 200, + "total_tokens": 2200 + }, + "after": { + "cache_read_detail_recorded_requests": 2, + "cache_write_detail_recorded_requests": 2, + "requests": 2, + "invalid_usage": 0, + "missing_usage": 0, + "prompt_tokens": 2000, + "completion_tokens": 200, + "total_tokens": 2200 + } + } + ], + "schema": "fleetdiff-litellm-spend/v1", + "token_accounting": "genai-default-accounting/v1+usage-provenance/v1", + "source_contract": "litellm-spend-rows/v1", + "group_by": "key", + "hashing": "environment", + "input_rows": 4, + "out_of_scope_rows": 0, + "outside_period_rows": 0, + "before": { + "cache": { + "requests": 2, + "read": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 1840, + "share_of_recorded_input": 0.92 + }, + "write": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 100, + "share_of_recorded_input": 0.05 + }, + "uncached_input_tokens": 160, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-07T00:00:00Z", + "end_exclusive": "2026-10-08T00:00:00Z" + }, + "logged_model_requests": 2, + "recorded_tokens": 2200, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 1840, + "gen_ai_sketch_cache_write_input_tokens_total": 100, + "gen_ai_sketch_input_tokens_total": 2000, + "gen_ai_sketch_output_tokens_total": 200, + "gen_ai_sketch_reasoning_output_tokens_total": 20, + "gen_ai_sketch_requests_total": 2, + "gen_ai_sketch_total_tokens_total": 2200 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 0 + }, + "recorded_spend_usd": "0.002", + "attributed_tokens": 2200, + "distinct_groups_estimate": 2.0000076294333358, + "distinct_nominal_rse": 0.008125 + }, + "after": { + "cache": { + "requests": 2, + "read": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 1220, + "share_of_recorded_input": 0.61 + }, + "write": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 150, + "share_of_recorded_input": 0.075 + }, + "uncached_input_tokens": 780, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-08T00:00:00Z", + "end_exclusive": "2026-10-09T00:00:00Z" + }, + "logged_model_requests": 2, + "recorded_tokens": 2200, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 1220, + "gen_ai_sketch_cache_write_input_tokens_total": 150, + "gen_ai_sketch_input_tokens_total": 2000, + "gen_ai_sketch_output_tokens_total": 200, + "gen_ai_sketch_reasoning_output_tokens_total": 20, + "gen_ai_sketch_requests_total": 2, + "gen_ai_sketch_total_tokens_total": 2200 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 0 + }, + "recorded_spend_usd": "0.002", + "attributed_tokens": 2200, + "distinct_groups_estimate": 2.0000076294333358, + "distinct_nominal_rse": 0.008125 + }, + "recorded_volume_split": { + "before_reported_tokens": 2200, + "after_reported_tokens": 2200, + "before_requests": 2, + "after_requests": 2, + "before_tokens_per_request": 1100, + "after_tokens_per_request": 1100, + "request_count_contribution_tokens": 0, + "tokens_per_request_contribution_tokens": 0 + }, + "models": [ + { + "item": "model-1", + "hash": "5b9c6b9597d1dd78b90e3932e3aaa05d0f4b9f533159ab69292d40fb82be57d8", + "before": { + "cache": { + "requests": 2, + "read": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 1840, + "share_of_recorded_input": 0.92 + }, + "write": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 100, + "share_of_recorded_input": 0.05 + }, + "uncached_input_tokens": 160, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-07T00:00:00Z", + "end_exclusive": "2026-10-08T00:00:00Z" + }, + "logged_model_requests": 2, + "recorded_tokens": 2200, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 1840, + "gen_ai_sketch_cache_write_input_tokens_total": 100, + "gen_ai_sketch_input_tokens_total": 2000, + "gen_ai_sketch_output_tokens_total": 200, + "gen_ai_sketch_reasoning_output_tokens_total": 20, + "gen_ai_sketch_requests_total": 2, + "gen_ai_sketch_total_tokens_total": 2200 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 0 + }, + "recorded_spend_usd": "0.002" + }, + "after": { + "cache": { + "requests": 2, + "read": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 1220, + "share_of_recorded_input": 0.61 + }, + "write": { + "coverage_known": true, + "reported_requests": 2, + "covered_requests": 2, + "missing_detail_requests": 0, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": 2000, + "tokens": 150, + "share_of_recorded_input": 0.075 + }, + "uncached_input_tokens": 780, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-08T00:00:00Z", + "end_exclusive": "2026-10-09T00:00:00Z" + }, + "logged_model_requests": 2, + "recorded_tokens": 2200, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 1220, + "gen_ai_sketch_cache_write_input_tokens_total": 150, + "gen_ai_sketch_input_tokens_total": 2000, + "gen_ai_sketch_output_tokens_total": 200, + "gen_ai_sketch_reasoning_output_tokens_total": 20, + "gen_ai_sketch_requests_total": 2, + "gen_ai_sketch_total_tokens_total": 2200 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 0 + }, + "recorded_spend_usd": "0.002" + }, + "recorded_volume_split": { + "before_reported_tokens": 2200, + "after_reported_tokens": 2200, + "before_requests": 2, + "after_requests": 2, + "before_tokens_per_request": 1100, + "after_tokens_per_request": 1100, + "request_count_contribution_tokens": 0, + "tokens_per_request_contribution_tokens": 0 + } + } + ], + "rankings": { + "name": "top_key", + "weight_unit": "attributed-recorded-tokens", + "before_weight": 2200, + "after_weight": 2200, + "before_max_error": 0, + "after_max_error": 0, + "candidate_count": 2, + "untracked_delta_bound": { + "lower": 0, + "upper": 0 + }, + "tracked_movers": [ + { + "item": "key-1", + "hash": "977a7cf5669d8e11", + "before": { + "lower": 1100, + "upper": 1100 + }, + "after": { + "lower": 1100, + "upper": 1100 + }, + "delta": { + "lower": 0, + "upper": 0 + }, + "direction": "unchanged" + }, + { + "item": "key-2", + "hash": "1dfcbfe4d3e713b9", + "before": { + "lower": 1100, + "upper": 1100 + }, + "after": { + "lower": 1100, + "upper": 1100 + }, + "delta": { + "lower": 0, + "upper": 0 + }, + "direction": "unchanged" + } + ] + }, + "zero_filled": "cannot determine from this export", + "provider_origin": "not declared by the supported SQL projection", + "notes": [ + "Periods are half-open UTC intervals selected by request startTime; rows crossing a boundary remain in their start period.", + "Counts describe supplied logged model requests. Export completeness and hidden provider retries are not established by spend rows.", + "Zero-only rows have unknown origin. Failures overlap usage categories; valid counts on failures remain recorded.", + "Aliases use local keyed hashes; session identities are scoped to their API key. A recorded session need not be an agent conversation.", + "Recorded spend is the exact decimal sum of usable source values in USD, not an invoice. Default-zero costs have unknown origin.", + "Rankings describe tracked candidates; concentration uses the net recorded increase and can exceed 100% when other contributors decreased." + ] +} diff --git a/internal/cli/testdata/import-refactor/cache-periods.text b/internal/cli/testdata/import-refactor/cache-periods.text new file mode 100644 index 0000000..537be0c --- /dev/null +++ b/internal/cli/testdata/import-refactor/cache-periods.text @@ -0,0 +1,32 @@ +Recorded tokens were unchanged (2,200 -> 2,200). +But cache reuse fell: cache-read share 92% -> 61% (uncached input +620). +Logged model requests: 2 -> 2. +Before: Wed 2026-10-07 00:00:00 UTC to Thu 2026-10-08 00:00:00 UTC (end exclusive). +After: Thu 2026-10-08 00:00:00 UTC to Fri 2026-10-09 00:00:00 UTC (end exclusive). + +What changed? + model-1 (5b9c6b9597d1dd78b90e3932e3aaa05d0f4b9f533159ab69292d40fb82be57d8): 2,200 -> 2,200 recorded tokens; 2 -> 2 requests. + 1,100.00 -> 1,100.00 recorded tokens per request; +0 from request count, +0 from request size. + Cache-read share of recorded input: 92% -> 61%. + Cache reads: 1,840 of 2,000 input tokens -> 1,220 of 2,000 (all requests report cache detail). + Cache-write share of recorded input: 5% -> 7.5%. + Cache writes: 100 of 2,000 input tokens -> 150 of 2,000 (all requests report cache detail). + Uncached input +620: cache-read share fell from 92% to 61%. + +Who drove the change? + key-1 (977a7cf5669d8e11): 1,100 -> 1,100 tokens; change +0. + key-2 (1dfcbfe4d3e713b9): 1,100 -> 1,100 tokens; change +0. + Attributed tokens: 2,200 -> 2,200. + +Whose uncached input grew? + Ranked by recorded uncached-input growth on read-covered requests; volume and workload mix can also change it. + key-1 (977a7cf5669d8e11): 80 -> 700 uncached input tokens; change +620. + Cache-read coverage: all requests report cache detail. + +How complete is the export? + Zero-filled: cannot determine from this export. Provider origin: not declared by the supported SQL projection. + Recorded spend (USD, analyzed calls): 0.002 -> 0.002. + No usage problems found. + Rows read: 4. + +Next: Use your provider's cache diagnostics (Anthropic, OpenAI) to find why, and check that your gateway route preserves cache markers. diff --git a/internal/cli/testdata/import-refactor/inspect.json b/internal/cli/testdata/import-refactor/inspect.json new file mode 100644 index 0000000..b7ab397 --- /dev/null +++ b/internal/cli/testdata/import-refactor/inspect.json @@ -0,0 +1,358 @@ +{ + "schema": "fleetdiff-inspect/v1", + "accounting_id": "genai-default-accounting/v1+usage-provenance/v1", + "hashing": "environment", + "capture": { + "files": 1, + "bytes": 3503, + "records": 1, + "spans": 6, + "dropped_attributes": 0 + }, + "metrics": { + "gen_ai_sketch_agent_runs_total": 1, + "gen_ai_sketch_cache_read_input_tokens_total": 25, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_dedup_key_missing_total": 0, + "gen_ai_sketch_dedup_suppressed_total": 0, + "gen_ai_sketch_input_tokens_total": 140, + "gen_ai_sketch_missing_token_usage_total": 1, + "gen_ai_sketch_output_tokens_total": 30, + "gen_ai_sketch_reasoning_output_tokens_total": 5, + "gen_ai_sketch_requests_total": 4, + "gen_ai_sketch_total_tokens_total": 170 + }, + "observed_counters": { + "gen_ai_sketch_agent_runs_total": 1, + "gen_ai_sketch_cache_read_input_tokens_total": 25, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_dedup_key_missing_total": 0, + "gen_ai_sketch_dedup_suppressed_total": 0, + "gen_ai_sketch_input_tokens_total": 140, + "gen_ai_sketch_missing_token_usage_total": 1, + "gen_ai_sketch_output_tokens_total": 30, + "gen_ai_sketch_reasoning_output_tokens_total": 5, + "gen_ai_sketch_requests_total": 4, + "gen_ai_sketch_total_tokens_total": 170 + }, + "metric_saturations": [], + "token_observations": { + "cache_read_input/conflict": 0, + "cache_read_input/invalid": 0, + "cache_read_input/missing": 0, + "cache_read_input/reported": 1, + "cache_read_input/subset_violation": 0, + "cache_write_input/conflict": 0, + "cache_write_input/invalid": 0, + "cache_write_input/missing": 0, + "cache_write_input/reported": 0, + "cache_write_input/subset_violation": 0, + "input/conflict": 0, + "input/invalid": 0, + "input/missing": 1, + "input/reported": 3, + "input/subset_violation": 0, + "output/conflict": 0, + "output/invalid": 0, + "output/missing": 1, + "output/reported": 3, + "output/subset_violation": 0, + "reasoning_output/conflict": 0, + "reasoning_output/invalid": 0, + "reasoning_output/missing": 0, + "reasoning_output/reported": 1, + "reasoning_output/subset_violation": 0 + }, + "usage_provenance": { + "input/inferred": 0, + "input/provider_reported": 0, + "input/unavailable": 0, + "input/unknown": 4, + "output/inferred": 0, + "output/provider_reported": 0, + "output/unavailable": 0, + "output/unknown": 4 + }, + "operation_mix": { + "agent": 1, + "model": 4, + "tool": 1 + }, + "readiness": { + "ready": 2, + "total": 7, + "questions": [ + { + "id": "model_activity", + "status": "ready", + "covered_attempts": 4, + "total_attempts": 4 + }, + { + "id": "token_usage", + "status": "partial", + "covered_attempts": 3, + "total_attempts": 4, + "next_step": "Record both input and output usage; resolve invalid, conflicting, or subset-violating values. Declare unavailable usage instead of filling in zeros." + }, + { + "id": "usage_origin", + "status": "cannot_determine", + "covered_attempts": 0, + "total_attempts": 4, + "next_step": "Add usage provenance at the provider/estimator hook. With no declaration, a real zero and a filled-in zero cannot be distinguished." + }, + { + "id": "distinct_users", + "status": "ready", + "covered_attempts": 4, + "total_attempts": 4 + }, + { + "id": "top_users_by_tokens", + "status": "partial", + "covered_attempts": 3, + "total_attempts": 4, + "next_step": "Include a user key and complete, consistent token usage on the same model attempts." + }, + { + "id": "session_rankings", + "status": "partial", + "covered_attempts": 2, + "total_attempts": 4, + "next_step": "Add gen_ai.conversation.id or session.id to model spans to unlock session rankings by attempts." + }, + { + "id": "prompt_rankings", + "status": "cannot_determine", + "covered_attempts": 0, + "total_attempts": 4, + "next_step": "Prompt rankings use gen_ai.request.prompt when present; retain your existing prompt collection policy." + } + ] + }, + "dimensions": [ + { + "attribute": "enduser.id", + "observations": 4, + "estimated_distinct_values": 4.000030517888555, + "nominal_rse": 0.008125, + "label_risk": "sensitive_identifier" + }, + { + "attribute": "attribute-1", + "name_hidden": true, + "observations": 2, + "estimated_distinct_values": 2.0000076294333358, + "nominal_rse": 0.008125, + "label_risk": "review" + }, + { + "attribute": "gen_ai.conversation.id", + "observations": 2, + "estimated_distinct_values": 2.0000076294333358, + "nominal_rse": 0.008125, + "label_risk": "sensitive_identifier" + }, + { + "attribute": "gen_ai.operation.name", + "observations": 4, + "estimated_distinct_values": 1.0000019073389315, + "nominal_rse": 0.008125, + "label_risk": "review" + }, + { + "attribute": "gen_ai.request.model", + "observations": 4, + "estimated_distinct_values": 1.0000019073389315, + "nominal_rse": 0.008125, + "label_risk": "review" + }, + { + "attribute": "service.name", + "observations": 4, + "estimated_distinct_values": 1.0000019073389315, + "nominal_rse": 0.008125, + "label_risk": "review" + } + ], + "identities": [ + { + "field": "user", + "present_attempts": 4, + "token_covered_attempts": 3, + "estimated_distinct": 4.000030517888555, + "nominal_rse": 0.008125, + "rankings": [ + { + "weight": "attempts", + "attributed_weight": 4, + "candidates": 4, + "max_error": 0, + "items": [ + { + "alias": "user-3", + "hash": "0f27320e0fd4a6c5", + "estimate": 1, + "lower_bound": 1, + "upper_bound": 1, + "lower_share": 0.25, + "upper_share": 0.25 + }, + { + "alias": "user-4", + "hash": "30bb17f1df47a172", + "estimate": 1, + "lower_bound": 1, + "upper_bound": 1, + "lower_share": 0.25, + "upper_share": 0.25 + }, + { + "alias": "user-1", + "hash": "b246afce563a3917", + "estimate": 1, + "lower_bound": 1, + "upper_bound": 1, + "lower_share": 0.25, + "upper_share": 0.25 + }, + { + "alias": "user-2", + "hash": "dd1826ab378d7ea8", + "estimate": 1, + "lower_bound": 1, + "upper_bound": 1, + "lower_share": 0.25, + "upper_share": 0.25 + } + ] + }, + { + "weight": "tokens", + "attributed_weight": 170, + "candidates": 2, + "max_error": 0, + "items": [ + { + "alias": "user-1", + "hash": "b246afce563a3917", + "estimate": 120, + "lower_bound": 120, + "upper_bound": 120, + "lower_share": 0.7058823529411765, + "upper_share": 0.7058823529411765 + }, + { + "alias": "user-2", + "hash": "dd1826ab378d7ea8", + "estimate": 50, + "lower_bound": 50, + "upper_bound": 50, + "lower_share": 0.29411764705882354, + "upper_share": 0.29411764705882354 + } + ] + } + ] + }, + { + "field": "session", + "present_attempts": 2, + "token_covered_attempts": 2, + "estimated_distinct": 2.0000076294333358, + "nominal_rse": 0.008125, + "rankings": [ + { + "weight": "attempts", + "attributed_weight": 2, + "candidates": 2, + "max_error": 0, + "items": [ + { + "alias": "session-1", + "hash": "4b4a9e332b761cf0", + "estimate": 1, + "lower_bound": 1, + "upper_bound": 1, + "lower_share": 0.5, + "upper_share": 0.5 + }, + { + "alias": "session-2", + "hash": "a9a1ff091bd5497b", + "estimate": 1, + "lower_bound": 1, + "upper_bound": 1, + "lower_share": 0.5, + "upper_share": 0.5 + } + ] + }, + { + "weight": "tokens", + "attributed_weight": 170, + "candidates": 2, + "max_error": 0, + "items": [ + { + "alias": "session-1", + "hash": "4b4a9e332b761cf0", + "estimate": 120, + "lower_bound": 120, + "upper_bound": 120, + "lower_share": 0.7058823529411765, + "upper_share": 0.7058823529411765 + }, + { + "alias": "session-2", + "hash": "a9a1ff091bd5497b", + "estimate": 50, + "lower_bound": 50, + "upper_bound": 50, + "lower_share": 0.29411764705882354, + "upper_share": 0.29411764705882354 + } + ] + } + ] + }, + { + "field": "prompt", + "present_attempts": 0, + "token_covered_attempts": 0, + "estimated_distinct": 0, + "nominal_rse": 0.008125, + "rankings": [ + { + "weight": "attempts", + "attributed_weight": 0, + "candidates": 0, + "max_error": 0, + "items": [] + }, + { + "weight": "tokens", + "attributed_weight": 0, + "candidates": 0, + "max_error": 0, + "items": [] + } + ] + } + ], + "next_steps": [ + "Record both input and output usage; resolve invalid, conflicting, or subset-violating values. Declare unavailable usage instead of filling in zeros.", + "Add usage provenance at the provider/estimator hook. With no declaration, a real zero and a filled-in zero cannot be distinguished.", + "Include a user key and complete, consistent token usage on the same model attempts.", + "Add gen_ai.conversation.id or session.id to model spans to unlock session rankings by attempts.", + "Prompt rankings use gen_ai.request.prompt when present; retain your existing prompt collection policy." + ], + "notes": [ + "Readiness describes fields in this capture, not completeness of traffic. Counts are model attempts; deduplication is off.", + "Distinct estimates describe this capture. A label's value count is not the total number of Prometheus series; other labels multiply combinations.", + "Ranking shares use attributed weight. Token rankings require both usage fields; attempt rankings also include missing-usage attempts.", + "Usage provenance is a declaration, not authenticated provider evidence. Token subsets are not added again.", + "Hashes are pseudonymous. Default aliases and ephemeral keys do not support identity matching between runs." + ] +} diff --git a/internal/cli/testdata/import-refactor/inspect.text b/internal/cli/testdata/import-refactor/inspect.text new file mode 100644 index 0000000..3c52537 --- /dev/null +++ b/internal/cli/testdata/import-refactor/inspect.text @@ -0,0 +1,54 @@ +Your traces can fully answer 2 of 7 questions. +Next: Record both input and output usage; resolve invalid, conflicting, or subset-violating values. Declare unavailable usage instead of filling in zeros. + +4 model attempts across 6 captured spans; 170 reported tokens. +Usage: 3 complete, 1 missing either field. +Origin unknown: input 4, output 4 attempts. + +Questions + Model activity: ready (4/4 attempts) + Token usage: partial (3/4 attempts) + Record both input and output usage; resolve invalid, conflicting, or subset-violating values. Declare unavailable usage instead of filling in zeros. + Usage origin: cannot determine (0/4 attempts) + Add usage provenance at the provider/estimator hook. With no declaration, a real zero and a filled-in zero cannot be distinguished. + Distinct users: ready (4/4 attempts) + Top users by tokens: partial (3/4 attempts) + Include a user key and complete, consistent token usage on the same model attempts. + Session rankings: partial (2/4 attempts) + Add gen_ai.conversation.id or session.id to model spans to unlock session rankings by attempts. + Prompt rankings: cannot determine (0/4 attempts) + Prompt rankings use gen_ai.request.prompt when present; retain your existing prompt collection policy. + +Label review: estimated distinct values in this capture + "enduser.id": ~4; 4 observations; sensitive identifier + "attribute-1": ~2; 2 observations; review + "gen_ai.conversation.id": ~2; 2 observations; sensitive identifier + "gen_ai.operation.name": ~1; 4 observations; review + "gen_ai.request.model": ~1; 4 observations; review + "service.name": ~1; 4 observations; review + Showing 6/6 fields; nominal HLL RSE 0.81%. Values are not total series. + +user: ~4 distinct; identity present on 4/4 attempts + By attempts: 4 attributed; 4/4 candidates shown + user-3 (0f27320e0fd4a6c5): 1 (25.00%) + user-4 (30bb17f1df47a172): 1 (25.00%) + user-1 (b246afce563a3917): 1 (25.00%) + user-2 (dd1826ab378d7ea8): 1 (25.00%) + By tokens: 170 attributed; 2/2 candidates shown + user-1 (b246afce563a3917): 120 (70.59%) + user-2 (dd1826ab378d7ea8): 50 (29.41%) + +session: ~2 distinct; identity present on 2/4 attempts + By attempts: 2 attributed; 2/2 candidates shown + session-1 (4b4a9e332b761cf0): 1 (50.00%) + session-2 (a9a1ff091bd5497b): 1 (50.00%) + By tokens: 170 attributed; 2/2 candidates shown + session-1 (4b4a9e332b761cf0): 120 (70.59%) + session-2 (a9a1ff091bd5497b): 50 (29.41%) + +Capture notes + Readiness describes fields in this capture, not completeness of traffic. Counts are model attempts; deduplication is off. + Distinct estimates describe this capture. A label's value count is not the total number of Prometheus series; other labels multiply combinations. + Ranking shares use attributed weight. Token rankings require both usage fields; attempt rankings also include missing-usage attempts. + Usage provenance is a declaration, not authenticated provider evidence. Token subsets are not added again. + Hashes are pseudonymous. Default aliases and ephemeral keys do not support identity matching between runs. diff --git a/internal/cli/testdata/import-refactor/session-review.json b/internal/cli/testdata/import-refactor/session-review.json new file mode 100644 index 0000000..1797b13 --- /dev/null +++ b/internal/cli/testdata/import-refactor/session-review.json @@ -0,0 +1,261 @@ +{ + "schema": "fleetdiff-litellm-session-budget/v1", + "grouping": "session_id_only", + "period": { + "start": "2026-10-03T00:00:00Z", + "end_exclusive": "2026-10-04T00:00:00Z" + }, + "automatic_period": false, + "observed_span": false, + "hashing": "environment", + "supplied_budget_usd": "1", + "counter_window_seconds": 3600, + "rows": 13, + "selected_rows": 13, + "unsupported_rows": 0, + "unattributed_rows": 0, + "unattributed_recorded_spend_usd": "0", + "unattributed_spend_rows": 0, + "unattributed_missing_spend_rows": 0, + "unattributed_invalid_spend_rows": 0, + "rejection_classified_rows": 13, + "failure_logging_completeness": "unknown", + "unclassified_failure_rows": 0, + "sessions_with_model_calls": 4, + "logged_model_calls_total": 12, + "rejection_only_sessions": 0, + "sessions_without_analyzed_calls": 0, + "singleton_sessions": 1, + "clipped_sessions": 0, + "possibly_clipped_sessions": 0, + "sessions_with_recorded_budget_or_iteration_rejection": 1, + "rejection_affected_spend_distribution_n": 1, + "rejection_affected_call_distribution_n": 1, + "complete_recorded_spend_usd": { + "n": 4, + "p50": "1.1", + "p95": "1.6", + "max": "1.6" + }, + "logged_model_calls": { + "n": 4, + "p50": "2", + "p95": "6", + "max": "6" + }, + "usage_quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 0 + }, + "recorded_tokens": 1320, + "recorded_rejections": { + "session_budget": 1 + }, + "largest_sessions": [ + { + "alias": "session-1", + "hash": "e44580be67556952c0a683f049d51594860b552101c3fbdf62c976883a2449dd", + "logged_model_calls": 3, + "recorded_tokens": 330, + "recorded_spend_usd": "1.6", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 4000, + "keys": 2, + "teams": 2, + "agents": 2 + }, + { + "alias": "session-2", + "hash": "cd6748b7e221e4fd9972a26c25a984c32e905762be496682c29da776ec064af6", + "logged_model_calls": 2, + "recorded_tokens": 220, + "recorded_spend_usd": "1.2", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 60, + "keys": 1, + "teams": 1, + "agents": 1 + }, + { + "alias": "session-3", + "hash": "c8026a54424cf2c6895ca2d5938365106b70e36c9e59d527c0af544ac3b500ab", + "logged_model_calls": 6, + "recorded_tokens": 660, + "recorded_spend_usd": "1.1", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": { + "session_budget": 1 + }, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": true, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 300, + "keys": 1, + "teams": 1, + "agents": 1 + }, + { + "alias": "session-4", + "hash": "99c75c498a76fbd74314425afc5af5389479fb28e8c3758c3d0726bbf56f8717", + "logged_model_calls": 1, + "recorded_tokens": 110, + "recorded_spend_usd": "0.2", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 0, + "keys": 1, + "teams": 1, + "agents": 1 + } + ], + "complete_sessions_exceeding_cap": 3, + "partial_subtotals_exceeding_cap": 0, + "exceeding_without_recorded_session_budget_rejection": 2, + "cap_review": [ + { + "alias": "session-1", + "hash": "e44580be67556952c0a683f049d51594860b552101c3fbdf62c976883a2449dd", + "logged_model_calls": 3, + "recorded_tokens": 330, + "recorded_spend_usd": "1.6", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 4000, + "keys": 2, + "teams": 2, + "agents": 2 + }, + { + "alias": "session-2", + "hash": "cd6748b7e221e4fd9972a26c25a984c32e905762be496682c29da776ec064af6", + "logged_model_calls": 2, + "recorded_tokens": 220, + "recorded_spend_usd": "1.2", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 60, + "keys": 1, + "teams": 1, + "agents": 1 + }, + { + "alias": "session-3", + "hash": "c8026a54424cf2c6895ca2d5938365106b70e36c9e59d527c0af544ac3b500ab", + "logged_model_calls": 6, + "recorded_tokens": 660, + "recorded_spend_usd": "1.1", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": { + "session_budget": 1 + }, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": true, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 300, + "keys": 1, + "teams": 1, + "agents": 1 + } + ], + "sessions_spanning_counter_window": 1, + "counter_window_review": [ + { + "alias": "session-1", + "hash": "e44580be67556952c0a683f049d51594860b552101c3fbdf62c976883a2449dd", + "logged_model_calls": 3, + "recorded_tokens": 330, + "recorded_spend_usd": "1.6", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 4000, + "keys": 2, + "teams": 2, + "agents": 2 + } + ], + "sessions_under_multiple_identities": 1, + "shared_counter_review": [ + { + "alias": "session-1", + "hash": "e44580be67556952c0a683f049d51594860b552101c3fbdf62c976883a2449dd", + "logged_model_calls": 3, + "recorded_tokens": 330, + "recorded_spend_usd": "1.6", + "spend_complete": true, + "missing_spend_rows": 0, + "invalid_spend_rows": 0, + "unanalyzed_rows": 0, + "recorded_rejections": {}, + "rejection_classification_complete": true, + "recorded_budget_or_iteration_rejection": false, + "clipped": false, + "possibly_clipped": false, + "positive_spend_span_seconds": 4000, + "keys": 2, + "teams": 2, + "agents": 2 + } + ] +} diff --git a/internal/cli/testdata/import-refactor/session-review.text b/internal/cli/testdata/import-refactor/session-review.text new file mode 100644 index 0000000..cdbfb01 --- /dev/null +++ b/internal/cli/testdata/import-refactor/session-review.text @@ -0,0 +1,36 @@ +3 of 4 sessions with complete recorded spend exceed the supplied USD 1.00 cap. +Grouped by session ID alone (LiteLLM v1.104.0 counter unit). Sat 2026-10-03 00:00:00 UTC to Sun 2026-10-04 00:00:00 UTC (end exclusive). + +Observed session totals +Recorded USD: p50 1.10; p95 1.60; max 1.60 (n=4, exact nearest-rank). +Logged model calls: p50 2; p95 6; max 6 (n=4, exact nearest-rank). +Sessions with recorded budget or iteration rejections: 1. Included in observed distributions: spend n=1, calls n=1. + +Largest sessions by recorded spend + session-1: USD 1.60; calls=3; hash e44580be67556952c0a683f049d51594860b552101c3fbdf62c976883a2449dd. + session-2: USD 1.20; calls=2; hash cd6748b7e221e4fd9972a26c25a984c32e905762be496682c29da776ec064af6. + session-3: USD 1.10; calls=6; recorded budget/iteration rejection; hash c8026a54424cf2c6895ca2d5938365106b70e36c9e59d527c0af544ac3b500ab. + session-4: USD 0.20; calls=1; hash 99c75c498a76fbd74314425afc5af5389479fb28e8c3758c3d0726bbf56f8717. + +Cap review +Above-cap sessions with no recorded session-budget rejection: 2. + session-1: USD 1.60; calls=3; hash e44580be67556952c0a683f049d51594860b552101c3fbdf62c976883a2449dd. + Recorded session-budget rejections: 0. + session-2: USD 1.20; calls=2; hash cd6748b7e221e4fd9972a26c25a984c32e905762be496682c29da776ec064af6. + Recorded session-budget rejections: 0. + session-3: USD 1.10; calls=6; recorded budget/iteration rejection; hash c8026a54424cf2c6895ca2d5938365106b70e36c9e59d527c0af544ac3b500ab. + Recorded session-budget rejections: 1. +Review last-call or parallel-call overshoot, the counter window, agent budget configuration and failure logging. Recorded totals alone do not establish enforcement behavior. + +Sessions spanning more than 3600 seconds between first and last positive-spend request starts: 1. + session-1: 4000 seconds. +This is a recorded duration, not evidence that a counter reset. + +Session IDs observed under multiple keys, teams or agents: 1. Each addresses the same counter key; shared storage and agent budget configuration determine actual tracking. + session-1: keys=2, teams=2, agents=2. + +Identity and usage coverage +12 logged model calls in the selected period. +1 of 4 identified sessions contain one model call. +Rejection classification: 13 of 13 selected rows include the derived column; failure logging completeness is unknown. + session_budget recorded rejections: 1. diff --git a/internal/cli/testdata/import-refactor/spend-periods.json b/internal/cli/testdata/import-refactor/spend-periods.json new file mode 100644 index 0000000..1b0b07c --- /dev/null +++ b/internal/cli/testdata/import-refactor/spend-periods.json @@ -0,0 +1,748 @@ +{ + "cache": { + "before": { + "requests": 8, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 8, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 8, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "after": { + "requests": 12, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 12, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 12, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "uncached_growth_rankings": null + }, + "comparison_incomplete": false, + "call_types": [ + { + "call_type": "acompletion", + "analyzed": true, + "before": { + "cache_read_detail_recorded_requests": 0, + "cache_write_detail_recorded_requests": 0, + "requests": 8, + "invalid_usage": 0, + "missing_usage": 0, + "prompt_tokens": 6400, + "completion_tokens": 1600, + "total_tokens": 8000 + }, + "after": { + "cache_read_detail_recorded_requests": 0, + "cache_write_detail_recorded_requests": 0, + "requests": 12, + "invalid_usage": 0, + "missing_usage": 0, + "prompt_tokens": 12800, + "completion_tokens": 3200, + "total_tokens": 16000 + } + } + ], + "schema": "fleetdiff-litellm-spend/v1", + "token_accounting": "genai-default-accounting/v1+usage-provenance/v1", + "source_contract": "litellm-spend-rows/v1", + "group_by": "session", + "hashing": "environment", + "input_rows": 20, + "out_of_scope_rows": 0, + "outside_period_rows": 0, + "before": { + "cache": { + "requests": 8, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 8, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 8, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-07T00:00:00Z", + "end_exclusive": "2026-10-08T00:00:00Z" + }, + "logged_model_requests": 8, + "recorded_tokens": 8000, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 0, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_input_tokens_total": 6400, + "gen_ai_sketch_output_tokens_total": 1600, + "gen_ai_sketch_reasoning_output_tokens_total": 0, + "gen_ai_sketch_requests_total": 8, + "gen_ai_sketch_total_tokens_total": 8000 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 2, + "failed_records": 1, + "unknown_status": 0, + "unclear_usage_records": 2, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 8 + }, + "recorded_spend_usd": "0", + "attributed_tokens": 8000, + "distinct_groups_estimate": 8.000122072795856, + "distinct_nominal_rse": 0.008125 + }, + "after": { + "cache": { + "requests": 12, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 12, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 12, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-08T00:00:00Z", + "end_exclusive": "2026-10-09T00:00:00Z" + }, + "logged_model_requests": 12, + "recorded_tokens": 16000, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 0, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_input_tokens_total": 12800, + "gen_ai_sketch_output_tokens_total": 3200, + "gen_ai_sketch_reasoning_output_tokens_total": 0, + "gen_ai_sketch_requests_total": 12, + "gen_ai_sketch_total_tokens_total": 16000 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 4, + "failed_records": 2, + "unknown_status": 0, + "unclear_usage_records": 4, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 12 + }, + "recorded_spend_usd": "0", + "attributed_tokens": 16000, + "distinct_groups_estimate": 12.000274666584165, + "distinct_nominal_rse": 0.008125 + }, + "models": [ + { + "item": "model-1", + "hash": "9e3b6340339f1799b4f8dde02b1230e56410938df6e7f91e1d4fc1b5ae0e7576", + "before": { + "cache": { + "requests": 4, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 4, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 4, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-07T00:00:00Z", + "end_exclusive": "2026-10-08T00:00:00Z" + }, + "logged_model_requests": 4, + "recorded_tokens": 4000, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 0, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_input_tokens_total": 3200, + "gen_ai_sketch_output_tokens_total": 800, + "gen_ai_sketch_reasoning_output_tokens_total": 0, + "gen_ai_sketch_requests_total": 4, + "gen_ai_sketch_total_tokens_total": 4000 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 4 + }, + "recorded_spend_usd": "0" + }, + "after": { + "cache": { + "requests": 6, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 6, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 6, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-08T00:00:00Z", + "end_exclusive": "2026-10-09T00:00:00Z" + }, + "logged_model_requests": 6, + "recorded_tokens": 12000, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 0, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_input_tokens_total": 9600, + "gen_ai_sketch_output_tokens_total": 2400, + "gen_ai_sketch_reasoning_output_tokens_total": 0, + "gen_ai_sketch_requests_total": 6, + "gen_ai_sketch_total_tokens_total": 12000 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 0, + "failed_records": 0, + "unknown_status": 0, + "unclear_usage_records": 0, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 6 + }, + "recorded_spend_usd": "0" + }, + "recorded_volume_split": { + "before_reported_tokens": 4000, + "after_reported_tokens": 12000, + "before_requests": 4, + "after_requests": 6, + "before_tokens_per_request": 1000, + "after_tokens_per_request": 2000, + "request_count_contribution_tokens": 3000, + "tokens_per_request_contribution_tokens": 5000 + } + }, + { + "item": "model-2", + "hash": "ee429eeac30892aebe572bbc91ea3d7eadfac7f7f3aa191152438d7f457f0cbb", + "before": { + "cache": { + "requests": 4, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 4, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 4, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-07T00:00:00Z", + "end_exclusive": "2026-10-08T00:00:00Z" + }, + "logged_model_requests": 4, + "recorded_tokens": 4000, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 0, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_input_tokens_total": 3200, + "gen_ai_sketch_output_tokens_total": 800, + "gen_ai_sketch_reasoning_output_tokens_total": 0, + "gen_ai_sketch_requests_total": 4, + "gen_ai_sketch_total_tokens_total": 4000 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 2, + "failed_records": 1, + "unknown_status": 0, + "unclear_usage_records": 2, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 4 + }, + "recorded_spend_usd": "0" + }, + "after": { + "cache": { + "requests": 6, + "read": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 6, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "write": { + "coverage_known": true, + "reported_requests": 0, + "covered_requests": 0, + "missing_detail_requests": 6, + "invalid_detail_or_input_requests": 0, + "zero_origin_unknown_requests": 0, + "covered_input_tokens": null, + "tokens": null, + "share_of_recorded_input": null + }, + "uncached_input_tokens": null, + "read_covered_requests_without_group_identity": 0 + }, + "period": { + "start": "2026-10-08T00:00:00Z", + "end_exclusive": "2026-10-09T00:00:00Z" + }, + "logged_model_requests": 6, + "recorded_tokens": 4000, + "counters": { + "gen_ai_sketch_cache_read_input_tokens_total": 0, + "gen_ai_sketch_cache_write_input_tokens_total": 0, + "gen_ai_sketch_input_tokens_total": 3200, + "gen_ai_sketch_output_tokens_total": 800, + "gen_ai_sketch_reasoning_output_tokens_total": 0, + "gen_ai_sketch_requests_total": 6, + "gen_ai_sketch_total_tokens_total": 4000 + }, + "quality": { + "missing_token_fields": 0, + "invalid_usage": 0, + "component_total_mismatch": 0, + "missing_total_field": 0, + "zero_only_origin_unknown": 4, + "failed_records": 2, + "unknown_status": 0, + "unclear_usage_records": 4, + "missing_group_identity": 0, + "missing_recorded_spend": 0, + "invalid_recorded_spend": 0, + "zero_spend_origin_unknown": 6 + }, + "recorded_spend_usd": "0" + } + } + ], + "rankings": { + "name": "top_session", + "weight_unit": "attributed-recorded-tokens", + "before_weight": 8000, + "after_weight": 16000, + "before_max_error": 0, + "after_max_error": 0, + "candidate_count": 14, + "untracked_delta_bound": { + "lower": 0, + "upper": 0 + }, + "tracked_movers": [ + { + "item": "session-1", + "hash": "816cb9397ca2361a", + "before": { + "lower": 2000, + "upper": 2000 + }, + "after": { + "lower": 0, + "upper": 0 + }, + "delta": { + "lower": -2000, + "upper": -2000 + }, + "direction": "decreased" + }, + { + "item": "session-2", + "hash": "55e2e9a9d1a469b5", + "before": { + "lower": 2000, + "upper": 2000 + }, + "after": { + "lower": 0, + "upper": 0 + }, + "delta": { + "lower": -2000, + "upper": -2000 + }, + "direction": "decreased" + }, + { + "item": "session-3", + "hash": "9436e6085b7809ac", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-4", + "hash": "92e20bf5a5587368", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-5", + "hash": "e8027ee8563364cf", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-6", + "hash": "270866b65e7c413f", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-7", + "hash": "8465fbb30bb71164", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-8", + "hash": "5d0c2121fb19a72c", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-9", + "hash": "6ea296250f8f9c1a", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-10", + "hash": "777f19a7137988bb", + "before": { + "lower": 0, + "upper": 0 + }, + "after": { + "lower": 2000, + "upper": 2000 + }, + "delta": { + "lower": 2000, + "upper": 2000 + }, + "direction": "increased" + }, + { + "item": "session-11", + "hash": "ad7f92e2521cb986", + "before": { + "lower": 1000, + "upper": 1000 + }, + "after": { + "lower": 0, + "upper": 0 + }, + "delta": { + "lower": -1000, + "upper": -1000 + }, + "direction": "decreased" + }, + { + "item": "session-12", + "hash": "9267800bdfe9cad8", + "before": { + "lower": 1000, + "upper": 1000 + }, + "after": { + "lower": 0, + "upper": 0 + }, + "delta": { + "lower": -1000, + "upper": -1000 + }, + "direction": "decreased" + }, + { + "item": "session-13", + "hash": "21df4eafedbd907b", + "before": { + "lower": 1000, + "upper": 1000 + }, + "after": { + "lower": 0, + "upper": 0 + }, + "delta": { + "lower": -1000, + "upper": -1000 + }, + "direction": "decreased" + }, + { + "item": "session-14", + "hash": "1efeb420e08f0d0f", + "before": { + "lower": 1000, + "upper": 1000 + }, + "after": { + "lower": 0, + "upper": 0 + }, + "delta": { + "lower": -1000, + "upper": -1000 + }, + "direction": "decreased" + } + ] + }, + "concentration_of_increase": { + "tracked_contributors": 2, + "recorded_token_delta": { + "lower": 4000, + "upper": 4000 + }, + "share_of_net_recorded_increase": { + "lower": 0.5, + "upper": 0.5 + } + }, + "zero_filled": "cannot determine from this export", + "provider_origin": "not declared by the supported SQL projection", + "notes": [ + "Periods are half-open UTC intervals selected by request startTime; rows crossing a boundary remain in their start period.", + "Counts describe supplied logged model requests. Export completeness and hidden provider retries are not established by spend rows.", + "Zero-only rows have unknown origin. Failures overlap usage categories; valid counts on failures remain recorded.", + "Aliases use local keyed hashes; session identities are scoped to their API key. A recorded session need not be an agent conversation.", + "Recorded spend is the exact decimal sum of usable source values in USD, not an invoice. Default-zero costs have unknown origin.", + "Rankings describe tracked candidates; concentration uses the net recorded increase and can exceed 100% when other contributors decreased." + ] +} diff --git a/internal/cli/testdata/import-refactor/spend-periods.text b/internal/cli/testdata/import-refactor/spend-periods.text new file mode 100644 index 0000000..1fc6ffb --- /dev/null +++ b/internal/cli/testdata/import-refactor/spend-periods.text @@ -0,0 +1,42 @@ +Recorded tokens doubled (8,000 -> 16,000). +model-1 accounts for 100% of the net recorded increase (+8,000 tokens). +Logged model requests: 8 -> 12. +Before: Wed 2026-10-07 00:00:00 UTC to Thu 2026-10-08 00:00:00 UTC (end exclusive). +After: Thu 2026-10-08 00:00:00 UTC to Fri 2026-10-09 00:00:00 UTC (end exclusive). + +What changed? + model-1 (9e3b6340339f1799b4f8dde02b1230e56410938df6e7f91e1d4fc1b5ae0e7576): 4,000 -> 12,000 recorded tokens; 4 -> 6 requests. + 1,000.00 -> 2,000.00 recorded tokens per request; +3,000 from request count, +5,000 from request size. + model-2 (ee429eeac30892aebe572bbc91ea3d7eadfac7f7f3aa191152438d7f457f0cbb): 4,000 -> 4,000 recorded tokens; 4 -> 6 requests. + Cache detail not recorded in this export. + +Who drove the increase? + session-1 (816cb9397ca2361a): 2,000 -> 0 tokens; change -2,000. + session-2 (55e2e9a9d1a469b5): 2,000 -> 0 tokens; change -2,000. + session-3 (9436e6085b7809ac): 0 -> 2,000 tokens; change +2,000. + session-4 (92e20bf5a5587368): 0 -> 2,000 tokens; change +2,000. + session-5 (e8027ee8563364cf): 0 -> 2,000 tokens; change +2,000. + session-6 (270866b65e7c413f): 0 -> 2,000 tokens; change +2,000. + session-7 (8465fbb30bb71164): 0 -> 2,000 tokens; change +2,000. + session-8 (5d0c2121fb19a72c): 0 -> 2,000 tokens; change +2,000. + session-9 (6ea296250f8f9c1a): 0 -> 2,000 tokens; change +2,000. + session-10 (777f19a7137988bb): 0 -> 2,000 tokens; change +2,000. + session-11 (ad7f92e2521cb986): 1,000 -> 0 tokens; change -1,000. + session-12 (9267800bdfe9cad8): 1,000 -> 0 tokens; change -1,000. + session-13 (21df4eafedbd907b): 1,000 -> 0 tokens; change -1,000. + session-14 (1efeb420e08f0d0f): 1,000 -> 0 tokens; change -1,000. + 2 leading tracked sessions account for 50% of the net recorded increase. + Attributed tokens: 8,000 -> 16,000. + +How complete is the export? + Records needing usage review (categories may overlap): 2 -> 4. + Zero-only records, origin unknown: 2 -> 4. + Failed records (overlap usage categories): 1 -> 2. + Zero-filled: cannot determine from this export. Provider origin: not declared by the supported SQL projection. + Zero-cost origin unknown: 8 -> 12. + No other usage problems found. + Rows read: 20. + Whole-period per-request split needs clearer usage; recorded totals remain shown. + model-2 per-request split needs clearer usage on 2 before / 4 after records. + +Next: review the leading contributors and flagged records in your local LiteLLM logs. diff --git a/internal/inspect/decode.go b/internal/inspect/decode.go index 310e6d3..3f1f327 100644 --- a/internal/inspect/decode.go +++ b/internal/inspect/decode.go @@ -14,6 +14,8 @@ import ( "io" "unicode/utf8" + "github.com/llm-measurement/fleetdiff/internal/boundedjson" + tracepb "go.opentelemetry.io/proto/slim/otlp/collector/trace/v1" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" @@ -101,16 +103,10 @@ func decodeJSON(data []byte) (*traceRequest, error) { if !utf8.Valid(data) { return nil, errEncoding } - dec := json.NewDecoder(bytes.NewReader(data)) - dec.UseNumber() - left := maxJSONNodes - value, err := readJSONValue(dec, 0, &left) + value, err := boundedjson.Decode(data, maxJSONDepth, maxJSONNodes) if err != nil { return nil, errEncoding } - if _, err = dec.Token(); !errors.Is(err, io.EOF) { - return nil, errEncoding - } root, ok := value.(map[string]any) if !ok { return nil, errEncoding @@ -204,58 +200,3 @@ func fixIDs(v any, parent bool) error { } return nil } - -func readJSONValue(d *json.Decoder, depth int, left *int) (any, error) { - *left-- - if depth > maxJSONDepth || *left < 0 { - return nil, errEncoding - } - t, err := d.Token() - if err != nil { - return nil, err - } - switch t { - case json.Delim('{'): - m := map[string]any{} - for d.More() { - t, err := d.Token() - if err != nil { - return nil, err - } - key, ok := t.(string) - if !ok { - return nil, errEncoding - } - if _, duplicate := m[key]; duplicate { - return nil, errEncoding - } - v, err := readJSONValue(d, depth+1, left) - if err != nil { - return nil, err - } - m[key] = v - } - if t, err := d.Token(); err != nil || t != json.Delim('}') { - return nil, errEncoding - } - return m, nil - case json.Delim('['): - a := []any{} - for d.More() { - v, err := readJSONValue(d, depth+1, left) - if err != nil { - return nil, err - } - a = append(a, v) - } - if t, err := d.Token(); err != nil || t != json.Delim(']') { - return nil, errEncoding - } - return a, nil - default: - if _, bad := t.(json.Delim); bad { - return nil, errEncoding - } - return t, nil - } -} diff --git a/internal/spend/accounting.go b/internal/spend/accounting.go index 31dfd84..f951fb8 100644 --- a/internal/spend/accounting.go +++ b/internal/spend/accounting.go @@ -5,7 +5,6 @@ package spend import ( "math/big" - "regexp" "strings" "github.com/llm-measurement/fleetdiff/internal/accounting" @@ -117,26 +116,12 @@ func (q *Quality) add(v Quality) { q.SpendZeroUnknown += v.SpendZeroUnknown } -var decimalPattern = regexp.MustCompile(`^[0-9]{1,19}(\.[0-9]{1,20})?([eE][+-]?[0-9]{1,3})?$`) -var maxCost = new(big.Rat).SetInt(new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)) - // SQL float values arrive as decimal text. Preserve that text's exact value, // including subnormal doubles, with bounded precision and no binary rounding. func costValue(s string) (*big.Rat, bool) { - if len(s) > 48 || !decimalPattern.MatchString(s) { - return nil, false - } - r, ok := new(big.Rat).SetString(s) - if !ok || r.Sign() < 0 || r.Cmp(maxCost) > 0 { - return nil, false - } - if digits, exact := r.FloatPrec(); !exact || digits > 344 { - return nil, false - } - return r, true + return accounting.Cost(s) } func costText(n *big.Rat) string { - digits, _ := n.FloatPrec() // Every accepted input has a terminating decimal. - return n.FloatString(digits) + return accounting.CostText(n) } diff --git a/internal/spend/identity_golden_test.go b/internal/spend/identity_golden_test.go new file mode 100644 index 0000000..97986d8 --- /dev/null +++ b/internal/spend/identity_golden_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package spend + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "os" + "strings" + "testing" +) + +// Freeze the legacy private helper's bytes, including canonicalization edges. +func TestLegacyIdentityGoldens(t *testing.T) { + type vector struct { + Kind string `json:"kind"` + Parts []string `json:"parts"` + Hash string `json:"hash"` + } + cases := []vector{ + {Kind: "session", Parts: []string{" Key A ", "S\r\nession"}}, + {Kind: "user", Parts: []string{"cafe\u0301"}}, + {Kind: "key", Parts: []string{"\u001c\u001d\u001e\u001f"}}, + {Kind: "team", Parts: []string{""}}, + {Kind: "session", Parts: []string{"a" + strings.Repeat("\u0301", 31), "\U00011f41\U00011f3e"}}, + } + key := []byte("synthetic-refactor-regression-key-32-bytes") + for i := range cases { + h, err := identity(key, cases[i].Kind, cases[i].Parts...) + if err != nil { + t.Fatal(err) + } + cases[i].Hash = hex.EncodeToString(h[:]) + } + got, err := json.MarshalIndent(cases, "", " ") + if err != nil { + t.Fatal(err) + } + got = append(got, '\n') + const path = "testdata/legacy-identity.json" + if os.Getenv("FLEETDIFF_UPDATE_REFACTOR_GOLDENS") == "1" { + if err := os.MkdirAll("testdata", 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, got, 0644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatal("identity bytes differ from v0.8.0") + } +} diff --git a/internal/spend/investigate.go b/internal/spend/investigate.go index 14d98fc..65fa23c 100644 --- a/internal/spend/investigate.go +++ b/internal/spend/investigate.go @@ -6,21 +6,19 @@ package spend import ( "bytes" "crypto/hmac" - "crypto/rand" "crypto/sha256" "encoding/binary" "encoding/json" "errors" "fmt" "math/big" - "os" "slices" "strings" "time" "github.com/llm-measurement/fleetdiff/internal/accounting" + "github.com/llm-measurement/fleetdiff/internal/attribution" "github.com/llm-measurement/fleetdiff/internal/compare" - "github.com/llm-measurement/llm-sketchkit/go/sketchkit/canon" "github.com/llm-measurement/llm-sketchkit/go/sketchkit/frequentitems" sketchhash "github.com/llm-measurement/llm-sketchkit/go/sketchkit/hash" "github.com/llm-measurement/llm-sketchkit/go/sketchkit/hllpp" @@ -101,40 +99,13 @@ func volume(a, b Window) *compare.VolumeChange { } func secret(env string) ([]byte, string, error) { - if env == "" { - b := make([]byte, 32) - if _, err := rand.Read(b); err != nil { - return nil, "", errors.New("cannot generate a local hashing secret") - } - return b, "ephemeral", nil - } - if len(env) > 128 || len(os.Getenv(env)) > 4096 { - return nil, "", errors.New("invalid hash secret environment setting") - } - if _, err := sketchhash.SecretFromEnv(env); err != nil { - return nil, "", errors.New("hash secret missing or weak; use a random secret of at least 16 bytes") - } - return []byte(os.Getenv(env)), "environment", nil + return attribution.Secret(env) } // This is the same registered-domain HMAC construction as sketchkit/inspect. // Length-framed components separate source and identity kinds inside that domain. func identity(key []byte, kind string, parts ...string) ([32]byte, error) { - canonical := make([]string, 0, len(parts)+2) - canonical = append(canonical, "litellm-spend/v1", kind) - for _, part := range parts { - b, err := canon.CanonicalizeString(canon.TextV1, part) - if err != nil { - return [32]byte{}, errors.New("identity is not valid UTF-8") - } - canonical = append(canonical, string(b)) - } - b, _ := json.Marshal(canonical) - mac := hmac.New(sha256.New, key) - mac.Write([]byte(sketchhash.UserV1)) - mac.Write([]byte{0}) - mac.Write(b) - return [32]byte(mac.Sum(nil)), nil + return attribution.Identity(key, "litellm-spend/v1", kind, parts...) } func requestDigest(key []byte, id string) [32]byte { diff --git a/internal/spend/testdata/legacy-identity.json b/internal/spend/testdata/legacy-identity.json new file mode 100644 index 0000000..3395b39 --- /dev/null +++ b/internal/spend/testdata/legacy-identity.json @@ -0,0 +1,39 @@ +[ + { + "kind": "session", + "parts": [ + " Key A ", + "S\r\nession" + ], + "hash": "0d01e6940379fe22153a72b76eff6594ef9e6d2101643e49eddfae120b93ddea" + }, + { + "kind": "user", + "parts": [ + "café" + ], + "hash": "6dd1f42a6bbade55c5e3eb01f84be297bca0a418ea8eb84609bbb9b4c58f37bb" + }, + { + "kind": "key", + "parts": [ + "\u001c\u001d\u001e\u001f" + ], + "hash": "b9d473b1960fd8d3e20b3ec2ee955172593efb969f01df8079eaea498d52ae88" + }, + { + "kind": "team", + "parts": [ + "" + ], + "hash": "047e68ccd2fea75e3d1a40d2c5548ac8d10dc26d82e59bc9dc1a2c0b7483ec24" + }, + { + "kind": "session", + "parts": [ + "á́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́", + "𑽁𑼾" + ], + "hash": "6432fb8e698142abecaccd8e196568ba5a99e3913f73474aabffd48aa85305d3" + } +] From f64d4bec6f7bba032c908839a33cae7b0125010a Mon Sep 17 00:00:00 2001 From: Vijay Date: Tue, 13 Oct 2026 18:55:46 -0400 Subject: [PATCH 2/3] feat: prototype Claude Code observed-record comparisons Signed-off-by: Vijay --- .github/workflows/ci.yml | 13 + examples/claude-code-logs/.gitignore | 1 + examples/claude-code-logs/LONG_RUNNING.md | 146 +++++++ examples/claude-code-logs/README.md | 257 +++++++++++ examples/claude-code-logs/capture.py | 268 ++++++++++++ .../collector-long-running.json | 25 ++ examples/claude-code-logs/collector.yaml | 107 +++++ examples/claude-code-logs/fixtures.py | 141 ++++++ examples/claude-code-logs/test_capture.py | 370 ++++++++++++++++ examples/claude-code-logs/test_rotation.py | 286 +++++++++++++ internal/clauderecords/accounting.go | 133 ++++++ internal/clauderecords/identity.go | 131 ++++++ internal/clauderecords/investigate.go | 290 +++++++++++++ internal/clauderecords/investigate_test.go | 402 ++++++++++++++++++ internal/clauderecords/period.go | 75 ++++ internal/clauderecords/period_test.go | 158 +++++++ internal/clauderecords/reader.go | 291 +++++++++++++ internal/clauderecords/report.go | 176 ++++++++ internal/cli/claude.go | 156 +++++++ internal/cli/claude_test.go | 119 ++++++ internal/cli/run.go | 40 +- scripts/restricted-smoke.sh | 19 +- 22 files changed, 3602 insertions(+), 2 deletions(-) create mode 100644 examples/claude-code-logs/.gitignore create mode 100644 examples/claude-code-logs/LONG_RUNNING.md create mode 100644 examples/claude-code-logs/README.md create mode 100644 examples/claude-code-logs/capture.py create mode 100644 examples/claude-code-logs/collector-long-running.json create mode 100644 examples/claude-code-logs/collector.yaml create mode 100644 examples/claude-code-logs/fixtures.py create mode 100644 examples/claude-code-logs/test_capture.py create mode 100644 examples/claude-code-logs/test_rotation.py create mode 100644 internal/clauderecords/accounting.go create mode 100644 internal/clauderecords/identity.go create mode 100644 internal/clauderecords/investigate.go create mode 100644 internal/clauderecords/investigate_test.go create mode 100644 internal/clauderecords/period.go create mode 100644 internal/clauderecords/period_test.go create mode 100644 internal/clauderecords/reader.go create mode 100644 internal/clauderecords/report.go create mode 100644 internal/cli/claude.go create mode 100644 internal/cli/claude_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7542947..9f884e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,18 @@ jobs: run: | go build -o bin/fleetdiff ./cmd/fleetdiff FLEETDIFF_BIN="$PWD/bin/fleetdiff" python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v + - name: Claude log import and capture fixtures + run: | + FLEETDIFF_CLAUDE_CAPTURE_CLI=1 FLEETDIFF_BIN="$PWD/bin/fleetdiff" \ + python3 -m unittest discover -s examples/claude-code-logs -p 'test_*.py' -v + - name: Pinned Claude log capture privacy + if: runner.os == 'Linux' + timeout-minutes: 10 + run: | + docker pull otel/opentelemetry-collector-contrib@sha256:39923a8e431bd1f57be82411999d389fcfe40857492e4365456d97a4c1f74be6 + docker pull python@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c + FLEETDIFF_CLAUDE_CAPTURE_DOCKER=1 FLEETDIFF_CLAUDE_CAPTURE_CLI=1 FLEETDIFF_BIN="$PWD/bin/fleetdiff" \ + python3 -m unittest discover -s examples/claude-code-logs -p 'test_*.py' -v - name: Pinned LiteLLM writer to read-only SQL export if: runner.os == 'Linux' timeout-minutes: 10 @@ -116,6 +128,7 @@ jobs: run: | go test ./internal/spend -run '^$' -fuzz '^FuzzReaderBoundedFormats$' -fuzztime 1000x -parallel 2 -timeout 2m go test ./internal/spend -run '^$' -fuzz '^FuzzSessionBudget$' -fuzztime 1000x -parallel 2 -timeout 2m + go test ./internal/clauderecords -run '^$' -fuzz '^FuzzClaudeEnvelope$' -fuzztime 10000x -parallel 2 -timeout 2m go test ./internal/compare -run '^$' -fuzz '^FuzzCanonicalSummary$' -fuzztime 10000x -parallel 2 -timeout 2m go test ./internal/compare -run '^$' -fuzz '^FuzzEmbeddedSketch$' -fuzztime 10000x -parallel 2 -timeout 2m go test ./internal/compare -run '^$' -fuzz '^FuzzScanSummary$' -fuzztime 10000x -parallel 2 -timeout 2m diff --git a/examples/claude-code-logs/.gitignore b/examples/claude-code-logs/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/examples/claude-code-logs/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/examples/claude-code-logs/LONG_RUNNING.md b/examples/claude-code-logs/LONG_RUNNING.md new file mode 100644 index 0000000..4ebf515 --- /dev/null +++ b/examples/claude-code-logs/LONG_RUNNING.md @@ -0,0 +1,146 @@ +# Organization Collector: Rotating Private Logs + +**Prototype, not production/account-auth conformance.** This route reuses the +exact redaction in [collector.yaml](collector.yaml); it does not extend the +300-second / 8-MiB try-it helper. Only operator-approved, private collection is +in scope. Observed records remain client-recorded, with finality and completeness +unknown. Default capture intentionally has no model labels or per-model breakdown. + +## Compose The Configuration + +Use contrib **v0.162.0**, image +`otel/opentelemetry-collector-contrib@sha256:39923a8e431bd1f57be82411999d389fcfe40857492e4365456d97a4c1f74be6`. +The JSON overlay is native collector configuration, not a separate processor: + +```sh +umask 077 +capture_parent=$(mktemp -d) +export CLAUDE_CAPTURE_DIR="$capture_parent/capture" +mkdir -m 700 "$CLAUDE_CAPTURE_DIR" +otelcol-contrib validate \ + --config=examples/claude-code-logs/collector.yaml \ + --config=examples/claude-code-logs/collector-long-running.json +``` + +This validates only; it does not start collection. Use an already-installed +verified binary, or the pinned cached container with the directory mounted at +the constant configured path. An approved service invocation uses the same two +`--config` arguments without `validate`. The overlay changes only the receiver +binding and exporter, preserving processors and their order. It binds HTTP/JSON +to `127.0.0.1:4318`, with the inherited 1-MiB receive limit. + +For an existing organization collector, incorporate the processor definitions +unchanged in a **dedicated Claude logs pipeline** and use its approved receiver. +Do not blindly merge the example `logs` pipeline over an existing pipeline. +The required order is `memory_limiter`, `filter/events`, `transform/privacy`, +`groupbyattrs/safe_envelope`, then this file exporter. Preserve existing unrelated +pipelines. Do not fan out raw input to a debug or other exporter before redaction. +Validate the fully merged configuration separately. This example does not test +your organization's fanout, remote authentication, TLS, or mixed-tenant receiver. +Loopback is suitable only for same-host senders; do not expose this unauthenticated +receiver to a network. No deployment or service changes are performed here. + +## Exact Policy And Limits + +The fixed active filename is `logs.jsonl`; only the writer's timestamped backup +names vary. There are no resource-derived paths, custom encoding or compression. + +| Setting | Value | Meaning at this pin | +| --- | --- | --- | +| `rotation.max_megabytes` | 8 | 8 * 1024 * 1024-byte size threshold; an oversized single write fails | +| `rotation.max_backups` | 15 | Retains up to 15 backup timestamps after asynchronous cleanup | +| `rotation.max_days` | 15 | Removes backups whose filename timestamps are over 15 * 24 hours old when cleanup runs | +| `rotation.localtime` | false | UTC backup timestamps | +| `append` | false | Required compatibility: `append: true` plus rotation is rejected | +| `create_directory` | false | Operator precreates and owns the private directory | + +The [pinned exporter config](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/ae8c507510f48f433ab47dd1c6b01a59d6c388b5/exporter/fileexporter/config.go) +and [writer factory](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/ae8c507510f48f433ab47dd1c6b01a59d6c388b5/exporter/fileexporter/factory.go) +are authoritative for these settings. Rotation uses +[timberjack v1.4.7](https://github.com/DeRuina/timberjack/blob/v1.4.7/timberjack.go) +(`Write`, `openExistingOrNew`, `openNew`, `millRunOnce`), pinned by the exporter's +[go.mod](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/ae8c507510f48f433ab47dd1c6b01a59d6c388b5/exporter/fileexporter/go.mod). +Its rotated filenames include a reason, such as +`logs-2026-10-13T15-49-19.182-size.jsonl`. Existing active files are reopened for +writing by the rotating writer; `append: false` does not mean restart truncation +on this branch. The exporter's normal `flush_interval` is ineffective with this +unbuffered rotation writer. + +**Do not interpret `max_days` as timed active-file rotation or strict record TTL.** +The exporter does not configure timberjack's time-rotation options. Cleanup +runs on writer activity/open/rotation, not an independent retention clock; +filename time is not each record's event time. A quiet active file can retain +old records indefinitely. The public README's age-rotation wording is therefore +not a strict TTL contract for this pinned implementation. + +With the dedicated naming pattern, the settled nominal file budget is +16 * 8 MiB = 128 MiB (active plus backups), **not a hard directory quota**. +Cleanup is asynchronous and may fail; unrelated files and copied snapshots are +outside that policy. Use an operator-enforced disk quota and alert on exporter +failures, disk usage and cleanup failure. High traffic can evict history before +15 days; no minimum comparison coverage is guaranteed. + +## Privacy And Retention Gates + +**Important limitation:** `factory.go` supplies file mode `0644`, and timberjack +`openNew` explicitly applies it with `chmod`, bypassing a restrictive umask. +Precreating one `0600` file does not preserve that mode through rotation. The +binary test confirms the next active file and later backups are `0644`. + +Require a dedicated, collector-owned `0700` directory and private ancestors, +no broad ACLs/shared mounts, and owner-only access to all copied snapshots. +`0644` inside this directory is an explicitly disclosed exception, not a +`0600` guarantee. If policy requires `0600` on every live file, **do not deploy +this rotating route**. A periodic chmod watcher is not an acceptable substitute. +Keep the tested short helper or await an exporter supporting a verified file-mode +setting. Redaction retains raw attribution IDs; it is not account authentication +or a detector for content disguised as an allowed identity string. + +The organization must approve its raw-ID retention period before enabling this +route. If it needs a strict TTL, its existing supervised retention system must +stop/quiesce collection, flush/close the writer, delete expired active/backups +and snapshots, then restart on a fresh private path, including during idle +periods. Do not use external copy-truncate or delete the active file while the +writer is live. This document does not install such a scheduler or certify it. +Absent that external lifecycle, only backup-age/count cleanup is provided. + +## Import A Frozen File Set + +Stop or quiesce the approved pipeline before freezing files; do not import the +active file during writes or race an asynchronous backup deletion. Record selected +file hashes, client version, capture profile, configuration hash and coverage +gaps privately. Apply owner-only permissions to the frozen review copy and delete +it under the approved retention policy. A snapshot copied elsewhere is not +covered by exporter cleanup. + +The prototype importer uses repeated `--claude-code-logs FILE` flags, +not shell concatenation, with aggregate limits across files. For example, on the +updated local checkout: + +```sh +GOTOOLCHAIN=local GOPROXY=off go run ./cmd/fleetdiff investigate \ + --claude-code-logs "$snapshot/older.jsonl" \ + --claude-code-logs "$snapshot/newer.jsonl" \ + --group-by user --format json +``` + +Here `$snapshot` denotes an operator-prepared private immutable snapshot, not a +directory this command creates. Omit periods only to request the labeled default: +two latest complete seven-day UTC periods within the observed span when possible, +otherwise equal observed-span halves. Both explicit period flags override it. +Calendar completeness is not export completeness. The CLI-enabled rotation suite +checks both default selections, repeated copied files and the actual rotated +synthetic files against the local-checkout binary. The importer reads one bounded +byte snapshot for period selection and accounting; its own tests cover aggregate +bounds and snapshot consistency. +See [Importer Inputs And Limits](README.md#importer-inputs-and-limits) before +selecting files. Do not infer semantic +uniqueness from filenames, prompt IDs, or sequence numbers. + +## Next Gate + +After account-auth and organization privacy/lifecycle conformance, the next +backend candidate is **operator-exported Loki query-range JSON**, not supported +input today. Its source mapping and equal-timestamp pagination need their own +fixtures and version pin. This configuration adds no Loki/Elastic/Datadog exporter, +adapter, credentials or network calls. diff --git a/examples/claude-code-logs/README.md b/examples/claude-code-logs/README.md new file mode 100644 index 0000000..75b934f --- /dev/null +++ b/examples/claude-code-logs/README.md @@ -0,0 +1,257 @@ +# Claude Code Logs: Private File Capture + +Compare client-recorded token usage and cost across two periods, grouped +by user or owner-and-session, with accounting coverage and comparison quality checks. + +This is a **local-checkout prototype**, not a capability of the released v0.8.0 +binary. Use the standard [installation and verification guide](../../docs/OPERATIONS.md#install-and-verify) +for releases; do not assume an installed release accepts `--claude-code-logs`. +The commands below use `go run` from this checkout after its importer is built. +Account-auth conformance remains open; synthetic account/email tests do not +establish authenticated client behavior. Keep both capture routes prototype-only. + +The unit is an **observed `api_request` record**, with **finality unknown**. +Error and retries-exhausted companion counts stay separate. Interruption can +produce populated but partial usage; cancellation can emit no API record. +Neither a successful transport nor a gateway success flag proves final usage, +complete traffic or provider billing. Prompts, sessions and sequence numbers +are not unique request keys. Exact-copy filtering is not semantic deduplication. + +## Try A Fresh Synthetic File + +From the repository root, using an already-installed Go toolchain and cached +dependencies (these commands deliberately disable downloads): + +```sh +umask 077 +demo=$(mktemp -d) +python3 -B examples/claude-code-logs/fixtures.py --output "$demo/logs.jsonl" +GOTOOLCHAIN=local GOPROXY=off go run ./cmd/fleetdiff investigate \ + --claude-code-logs "$demo/logs.jsonl" \ + --before-period 2026-10-11T00:00:00Z/2026-10-12T00:00:00Z \ + --after-period 2026-10-12T00:00:00Z/2026-10-13T00:00:00Z \ + --group-by user --format json +``` + +The deterministic generator uses invented identities and emits one request in +each period with fictional model label `synthetic-model-a`, plus two error +companions in the second. This is not an available-model claim. Each request has ordinary +input 7, cache read 11, cache creation 5 and output 4: inclusive input 23 and +total 27. Decimal cost is `0.00003435`, not the rounded/truncated micros field 34. +Missing cache/usage fields are not zero. The generator is not a native-client +capture and does not provide provider-reported provenance. + +The example pins both disjoint, equal-duration periods. Select `--group-by +session` for owner-and-session attribution; session IDs alone are not global +identities. User attribution prefers account UUID, then email, then installation +ID, never a people count. Conflicting event/resource identities remain visible +for the importer to reject. Prompt quality uses owner, session and prompt IDs. +Add `--deduplicate-exact` only to request exact copied-record filtering, and use +`--format text` for the text report. Without a persistent key, hashes share only +this invocation's ephemeral scope. `--hash-secret-env VARIABLE` requires +`--source-scope APP`; provide secrets through your existing secret-management +process, not literals in commands, fixtures or source control. + +## Importer Inputs And Limits + +The prototype accepts repeated +`--claude-code-logs FILE` flags. With no period flags, it selects the latest two +complete seven-day UTC periods within the observed timestamp span when possible; +otherwise it labels two equal observed-span halves. Neither selection proves +complete export coverage. Supply both period flags to override; a single timestamp +cannot form an inferred comparison. The original capture suite uses explicit +periods; the additional rotation suite checks both inferred selections and +repeated-file copy handling against the local-checkout binary. These smoke checks +do not replace importer tests for aggregate limits and byte-snapshot consistency. + +Importer limits: **256 files**, **128 MiB across the file set**, +**8 MiB/OTLP envelope**, **1,000,000 log records across files**, +**250,000 each of attributed groups and scoped prompts**, and **128 distinct values +each for models, client versions and capture profiles**. Rotation does not increase +these aggregate limits. Use an immutable, stopped snapshot of the selected files. + +## Capture Route 1 Only + +Prerequisites: Python 3.10+, local Docker Engine 28+ / current Docker Desktop, +and these two images already installed. The helper uses `--pull never` and fails +if either image is absent; it never downloads dependencies or images. + +| Role | Pin | +| --- | --- | +| Collector Contrib v0.162.0 | `otel/opentelemetry-collector-contrib@sha256:39923a8e431bd1f57be82411999d389fcfe40857492e4365456d97a4c1f74be6` | +| Stdlib transport relay | `python@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c` | + +```sh +umask 077 +capture_parent=$(mktemp -d) +python3 -B examples/claude-code-logs/capture.py "$capture_parent/capture" --seconds 30 +``` + +The temporary parent works on Linux and macOS; its `capture` child must not yet +exist. Choose a **new directory outside the checkout**. The helper prints a random +`http://127.0.0.1:PORT/v1/logs` endpoint. For an independently authorized client +session, point only its logs exporter at that endpoint, use `http/json`, and +keep prompt, assistant, tool-content and raw-body logging off. Do not enable +enhanced beta tracing for this recipe. Example settings, with the printed port: + +```sh +export CLAUDE_CODE_ENABLE_TELEMETRY=1 +export OTEL_LOGS_EXPORTER=otlp +export OTEL_EXPORTER_OTLP_PROTOCOL=http/json +export OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/json +export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://127.0.0.1:PORT/v1/logs +export OTEL_LOG_USER_PROMPTS=0 OTEL_LOG_ASSISTANT_RESPONSES=0 +export OTEL_LOG_TOOL_DETAILS=0 OTEL_LOG_TOOL_CONTENT=0 OTEL_LOG_RAW_API_BODIES=0 +unset CLAUDE_CODE_ENHANCED_TELEMETRY_BETA +``` + +This does not start Claude or make model calls. Restore the client's previous +settings after the authorized interval; do not replace an existing production +export destination without approval. The recipe supports only OTLP HTTP/JSON +logs, without compression or chunked transfer, at most 1 MiB per request. It is +not an OTLP gRPC/protobuf, traces, metrics, Loki, Elastic or Datadog adapter. + +The collector and relay share an **internal Docker network** with no published +container ports and no external exporter. A serial host stdlib HTTP relay binds +only `127.0.0.1` and passes bounded request bodies through `docker exec` stdin; +it does not persist or print them. This accommodates Docker's internal-network +port-publishing restriction without adding an egress-enabled network. Raw data +exists in process memory before filtering. Local processes with access to the +port or Docker daemon are trusted; loopback is not authentication or a security +boundary against a compromised host. No credentials are required by this receiver. + +## Privacy and Retention Contract + +The [collector configuration](collector.yaml) applies these steps **before file +export**, failing the pipeline on transformation errors: + +1. Retain only exact body strings `claude_code.api_request`, + `claude_code.api_error` and `claude_code.api_retries_exhausted`. Unknown, map, + array or arbitrary text bodies are dropped, even with an API-like attribute. + Conflicting body/event-name pairs are dropped. +2. Keep only typed numeric accounting/sequence/status fields and approved + attribution strings at log level. Remove raw errors, prompts, code, paths, + model/query-source labels and unknown attributes. Wrong-type values are removed, + not coerced to zero; numeric semantic validation belongs to the importer. +3. At resource level keep `service.version` only in bounded numeric `X.Y.Z` form + and preserve `user.account_uuid`, `user.email`, `user.id`, `session.id` strings. + Set controlled `service.name=claude-code` and, **after allowlisting**, + `fleetdiff.capture.profile=claude-code-logs/v1`. The profile is a capture marker, + not an assertion that the client version was independently verified. +4. Keep no scope attributes. Replace scope name/version, clear severity text and + trace/span IDs, and set top-level event name from the accepted body. Pinned + `groupbyattrs` compaction rebuilds resource/scope envelopes without schema URLs; + the poisoned-URL test guards this version-specific behavior. + +At both log and resource locations, any present `identity.source` is replaced +with the fixed string `unverified` and retained. Before removing an existing +wrong-type, empty or whitespace-only account UUID, email or installation ID, +the same marker is set. Even an empty OTLP value counts as present. Missing +identity fields alone do not set it. This prevents a deleted malformed preferred +identity from silently falling back to installation attribution; the importer +suppresses attribution for the unverified regime. Raw OIDC/custom regime content +is never copied into the capture. + +Account-auth coverage remains unverified: the native mock evidence uses +installation attribution, and the account/email precedence and conflict checks +use synthetic identities. Neither proves field availability or account ownership +in authenticated Claude modes. No OIDC attribution profile is conformed. + +Raw account/email/installation/session/prompt IDs are deliberately retained for +attribution, so **the capture remains private**, including values at both event +and resource locations. This is a field-based contract, not a semantic detector +for a sender hiding content inside an allowed identity string. Resource version +and profile changes are exposed by importer aliases and comparison warnings; +an absent marker means unknown. Capture filtering and importer-output privacy +are separate obligations. Do not share `logs.jsonl` or a poisoned test input. + +The helper creates a `0700` directory and exclusively precreates `logs.jsonl` as +`0600`; `capture.json` is also `0600`. No rotation, dynamic filenames, compression +or appending to an old capture is used. Docker's hard file-size limit is **8 MiB**; +the timer is 1-300 seconds. Normal stop, SIGINT and SIGTERM stop/flush and remove +the containers/network. A cap hit or unexpected collector exit fails the helper; +the last JSON record can be partial, so do not silently salvage it as complete. +The manifest includes hashes, stop reason, retained-record count and unknown +finality/completeness. It is not a delivery-loss audit. Never import while writing. + +Import the stopped `logs.jsonl` with the command above and periods appropriate to +that file. Keep captures only for the approved review interval, **at most 24 hours**, +then delete both files, the capture directory and its now-empty temporary parent. +Deletion is an operator obligation, +not an automatic TTL; do not leave a capture helper unattended. SIGKILL, host +failure or Docker failure can prevent cleanup: inspect and remove only this +invocation's `fleetdiff-claude-*` resources. Finite capture does not imply complete +window coverage or deletion of telemetry already sent to another backend. + +## Existing Organization Collector + +For longer collection, use the [long-running composition and operator gates](LONG_RUNNING.md) +with [collector-long-running.json](collector-long-running.json), layered after +the same redaction configuration. The short helper above remains limited to +300 seconds / 8 MiB. The long-running policy uses 8 MiB files, 15 backups and +15-day backup-age cleanup; it is not a time-triggered rotation or strict TTL. +At this pin rotated files become `0644` inside the required `0700` directory. +Strict per-file `0600` policies block this route. Account-auth conformance and +organization-specific receiver authentication remain deployment gates. + +## Next Backend, Not Supported + +**Loki operator-exported JSON is the next candidate**, not an available adapter. +Its [query-range API](https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time) +provides a concrete stream export contract, but [OTLP ingestion mapping](https://grafana.com/docs/loki/latest/send-data/otel/) +changes field locations/names. Before support, pin a deployed version and prove +identity/profile locality, numeric precision, structured metadata, pagination and +same-timestamp boundary coverage using sanitized fixtures. No backend credentials, +queries or support are added here; Elastic and Datadog remain deferred. + +## Verify Without A Client Or Provider + +```sh +python3 -B examples/claude-code-logs/test_capture.py +FLEETDIFF_CLAUDE_CAPTURE_DOCKER=1 \ + python3 -B examples/claude-code-logs/test_capture.py +``` + +The first command runs local checks and explicitly skips Docker checks. The +second uses the exact pinned collector and locally cached relay image, invented +OTLP fixtures and no real client/provider. It tests poisoned resource/scope +attributes and URLs, body rejection, typed values, profile insertion, preserved +identity conflicts, missing versus zero, duplicate retention, loopback/internal +isolation, file permissions, the hard cap, timer and signal cleanup. Private +evidence and hashes are created outside git; `FLEETDIFF_CAPTURE_EVIDENCE` may +select a **new** private output directory. Importer-output sentinel tests are +maintained separately with the importer, not duplicated here. + +For a smoke check against the built importer in this checkout, also set +`FLEETDIFF_CLAUDE_CAPTURE_CLI=1` and `FLEETDIFF_BIN` pointing to the built binary. +Without `FLEETDIFF_BIN`, it falls back to local `go run` (default proxy off), +preserving supplied `GOTOOLCHAIN`, `GOCACHE` and checksum settings; a missing cache +path defaults inside the private temporary evidence directory. It checks +the fresh model fixture and the actual filtered file, including user/session and +JSON/text modes plus optional exact-copy filtering. The clean fixture exercises +model comparison through direct file import only. The default privacy capture +intentionally removes model labels and therefore has no per-model breakdown: +it reports missing model coverage instead of retaining custom model strings. +It does not fabricate a model for those records. + +The separate long-running checks exercise the exact rotation policy with fresh +synthetic logs and timestamped backup seeds, not a 15-day soak: + +```sh +FLEETDIFF_CLAUDE_CAPTURE_DOCKER=1 \ + python3 -B examples/claude-code-logs/test_rotation.py +``` + +They verify pinned-binary configuration acceptance, rejection of append plus +rotation, two real size rotations, backup-age/count cleanup, redaction, JSON +record preservation and the `0644` limitation. Evidence is private and outside +git, using the same `FLEETDIFF_CAPTURE_EVIDENCE` convention. Also set +`FLEETDIFF_CLAUDE_CAPTURE_CLI=1` and `FLEETDIFF_BIN` to the updated binary to test +multi-file import of the actual rotated files, copied-file retention versus exact +filtering, and both default-period selections. No credentials or client are used. + +Primary references: [pinned transform processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.162.0/processor/transformprocessor/README.md), +[filter processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.162.0/processor/filterprocessor/README.md), +[groupbyattrs log processing](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.162.0/processor/groupbyattrsprocessor/processor.go), +[file exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.162.0/exporter/fileexporter/README.md), +and [Claude Code monitoring](https://code.claude.com/docs/en/monitoring-usage). diff --git a/examples/claude-code-logs/capture.py b/examples/claude-code-logs/capture.py new file mode 100644 index 0000000..0617750 --- /dev/null +++ b/examples/claude-code-logs/capture.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: Apache-2.0 +# Code authors: Vijay and Codex +"""Bounded local capture with an already-installed collector; Python stdlib only.""" +import argparse +import hashlib +import http.client +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid + +ROOT = Path(__file__).resolve().parent +IMAGE = "otel/opentelemetry-collector-contrib@sha256:39923a8e431bd1f57be82411999d389fcfe40857492e4365456d97a4c1f74be6" +RELAY_IMAGE = "python@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c" +MAX_BYTES = 8 * 1024 * 1024 +MAX_REQUEST = 1024 * 1024 +RELAY = '''import sys, urllib.request +request = urllib.request.Request("http://127.0.0.1:4318/v1/logs", data=sys.stdin.buffer.read(1048577), + headers={"Content-Type": sys.argv[1]}) +opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) +with opener.open(request, timeout=3) as response: + sys.stdout.buffer.write(response.read(65536)) +''' + + +def docker(*args): + result = subprocess.run(["docker", *map(str, args)], capture_output=True, text=True, timeout=30) + if result.returncode: + raise RuntimeError("Local collector operation failed; no image was downloaded.") + return result.stdout.strip() + + +def send(endpoint, data): + request = urllib.request.Request(endpoint, data=data, headers={"Content-Type": "application/json"}) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(request, timeout=2) as response: + value = json.loads(response.read(65536)) + partial = value.get("partialSuccess", {}) + if partial.get("rejectedLogRecords", "0") not in (0, "0") or partial.get("errorMessage"): + raise RuntimeError("Collector reported partial acceptance.") + + +class Capture: + def __init__(self, directory, max_bytes=MAX_BYTES): + self.directory = Path(directory).absolute() + self.max_bytes = max_bytes + self.name = "fleetdiff-claude-" + uuid.uuid4().hex + self.network_created = False + self.container_created = False + self.sender_created = False + self.server = None + self.thread = None + self.endpoint = None + + def start(self): + docker("image", "inspect", IMAGE) + docker("image", "inspect", RELAY_IMAGE) + self.directory.mkdir(mode=0o700) + os.chmod(self.directory, 0o700) + with (self.directory / "logs.jsonl").open("xb") as output: + os.fchmod(output.fileno(), 0o600) + try: + docker("network", "create", "--internal", self.name) + self.network_created = True + docker("create", "--name", self.name, "--pull", "never", "--network", self.name, + "--user", f"{os.getuid()}:{os.getgid()}", "--read-only", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--memory=256m", "--pids-limit=128", + "--ulimit", f"fsize={self.max_bytes}:{self.max_bytes}", "--log-driver=none", + "--mount", f"type=bind,src={ROOT / 'collector.yaml'},dst=/etc/capture.yaml,readonly", + "--mount", f"type=bind,src={self.directory},dst=/capture", IMAGE, + "--config=/etc/capture.yaml") + self.container_created = True + docker("start", self.name) + docker("create", "--name", self.name + "-relay", "--pull", "never", + "--network", "container:" + self.name, "--user", f"{os.getuid()}:{os.getgid()}", + "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--memory=64m", "--pids-limit=32", "--log-driver=none", RELAY_IMAGE, + "python", "-B", "-c", "import time; time.sleep(600)") + self.sender_created = True + docker("start", self.name + "-relay") + self.start_relay() + deadline = time.monotonic() + 15 + while True: + try: + send(self.endpoint, b'{"resourceLogs":[]}') + return self.endpoint + except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.RemoteDisconnected): + if time.monotonic() >= deadline or not self.running(): + raise RuntimeError("Capture collector did not become ready.") from None + time.sleep(0.1) + except BaseException: + self.stop() + raise + + def start_relay(self): + session = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def setup(self): + super().setup() + self.connection.settimeout(5) + + def do_POST(self): + status, output = 400, b'{}' + try: + size = int(self.headers.get("Content-Length", "-1")) + content_type = self.headers.get("Content-Type", "") + if (self.path != "/v1/logs" or self.headers.get("Content-Encoding") + or self.headers.get("Transfer-Encoding") or not 0 <= size <= MAX_REQUEST + or content_type != "application/json"): + status = 413 + else: + data = self.rfile.read(size) + if len(data) == size: + result = subprocess.run(["docker", "exec", "-i", session.name + "-relay", + "python", "-B", "-c", RELAY, content_type], input=data, + capture_output=True, timeout=8) + status = 200 if result.returncode == 0 else 502 + # Only the response status is exposed, never collector error text. + if status == 200: + partial = json.loads(result.stdout or b'{}').get("partialSuccess", {}) + if partial.get("rejectedLogRecords", "0") not in (0, "0") or partial.get("errorMessage"): + status = 502 + except (ValueError, OSError, subprocess.SubprocessError): + status = 502 + self.send_response(status) + self.send_header("Content-Length", str(len(output))) + self.end_headers() + try: + self.wfile.write(output) + except OSError: + pass + + self.server = HTTPServer(("127.0.0.1", 0), Handler) + self.endpoint = f"http://127.0.0.1:{self.server.server_port}/v1/logs" + self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True) + self.thread.start() + + def running(self): + return docker("inspect", "--format", "{{.State.Running}}", self.name) == "true" + + def stop(self): + errors = [] + if self.server: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=10) + self.server = None + if self.sender_created: + try: + docker("rm", "--force", self.name + "-relay") + self.sender_created = False + except RuntimeError as error: + errors.append(error) + if self.container_created: + try: + docker("stop", "--time", "10", self.name) + except RuntimeError as error: + errors.append(error) + try: + docker("rm", "--force", self.name) + self.container_created = False + except RuntimeError as error: + errors.append(error) + if self.network_created: + try: + docker("network", "rm", self.name) + self.network_created = False + except RuntimeError as error: + errors.append(error) + if errors: + raise RuntimeError("Capture cleanup failed; check local Docker resources.") + + def finalize(self, reason): + path = self.directory / "logs.jsonl" + size = path.stat().st_size + valid = size < self.max_bytes and reason not in ("size-cap", "collector-stopped") + records = 0 + try: + for line in path.read_bytes().splitlines(): + batch = json.loads(line) + records += sum(len(scope.get("logRecords", [])) + for resource in batch.get("resourceLogs", []) + for scope in resource.get("scopeLogs", [])) + except (ValueError, TypeError, AttributeError): + valid = False + private = (stat.S_IMODE(path.stat().st_mode) == 0o600 + and stat.S_IMODE(self.directory.stat().st_mode) == 0o700) + metadata = {"profile": "claude-code-logs/v1", "image": IMAGE, "relay_image": RELAY_IMAGE, + "config_sha256": hashlib.sha256((ROOT / "collector.yaml").read_bytes()).hexdigest(), + "capture_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "bytes": size, "byte_cap": self.max_bytes, "observed_records": records, + "stop_reason": reason, "valid_jsonl_below_cap": valid, "private_modes": private, + "finality": "unknown", "completeness": "unknown"} + with (self.directory / "capture.json").open("x", encoding="ascii") as output: + os.fchmod(output.fileno(), 0o600) + json.dump(metadata, output, indent=2) + output.write("\n") + if not valid or not private or reason == "collector-stopped": + raise RuntimeError("Capture is incomplete or invalid; do not import it as a complete window.") + return metadata + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path, help="new private directory outside the repository") + parser.add_argument("--seconds", type=int, default=30, help="capture duration, 1-300 seconds") + args = parser.parse_args() + if not 1 <= args.seconds <= 300: + parser.error("duration must be 1-300 seconds") + if ROOT.parents[1] in args.directory.resolve().parents: + parser.error("capture must be outside the repository") + os.umask(0o077) + session = Capture(args.directory) + interrupted = False + + def interrupt(_signum, _frame): + nonlocal interrupted + interrupted = True + + for signum in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(signum, interrupt) + reason = "timer" + exit_code = 0 + try: + endpoint = session.start() + print(f"OTLP logs endpoint: {endpoint}", flush=True) + print(f"Capture stops after {args.seconds} seconds; raw attribution IDs remain private.", flush=True) + deadline = time.monotonic() + args.seconds + while time.monotonic() < deadline and not interrupted: + if not session.running(): + reason = "collector-stopped" + break + if (session.directory / "logs.jsonl").stat().st_size >= MAX_BYTES: + reason = "size-cap" + break + time.sleep(0.2) + if interrupted: + reason = "signal" + session.stop() + session.finalize(reason) + print("Capture stopped and flushed. Review privately, then delete the capture directory.") + except (OSError, RuntimeError, subprocess.SubprocessError): + print("Capture failed. Keep its files private; do not assume complete coverage.", file=sys.stderr) + exit_code = 1 + finally: + try: + session.stop() + except (RuntimeError, subprocess.SubprocessError): + print("Capture cleanup failed; check local Docker resources.", file=sys.stderr) + exit_code = 1 + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/claude-code-logs/collector-long-running.json b/examples/claude-code-logs/collector-long-running.json new file mode 100644 index 0000000..8a39e69 --- /dev/null +++ b/examples/claude-code-logs/collector-long-running.json @@ -0,0 +1,25 @@ +{ + "receivers": { + "otlp": { + "protocols": { + "http": { + "endpoint": "127.0.0.1:4318" + } + } + } + }, + "exporters": { + "file/capture": { + "path": "${env:CLAUDE_CAPTURE_DIR}/logs.jsonl", + "format": "json", + "append": false, + "create_directory": false, + "rotation": { + "max_megabytes": 8, + "max_days": 15, + "max_backups": 15, + "localtime": false + } + } + } +} diff --git a/examples/claude-code-logs/collector.yaml b/examples/claude-code-logs/collector.yaml new file mode 100644 index 0000000..22fcca5 --- /dev/null +++ b/examples/claude-code-logs/collector.yaml @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# Code authors: Vijay and Codex +# Run with capture.py: it supplies isolation, file permissions and a hard size cap. +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 1048576 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 128 + spike_limit_mib: 32 + filter/events: + error_mode: propagate + logs: + log_record: + - 'not IsString(body)' + - 'body != "claude_code.api_request" and body != "claude_code.api_error" and body != "claude_code.api_retries_exhausted"' + - 'attributes["event.name"] != nil and body == "claude_code.api_request" and attributes["event.name"] != "api_request"' + - 'attributes["event.name"] != nil and body == "claude_code.api_error" and attributes["event.name"] != "api_error"' + - 'attributes["event.name"] != nil and body == "claude_code.api_retries_exhausted" and attributes["event.name"] != "api_retries_exhausted"' + transform/privacy: + error_mode: propagate + log_statements: + - context: resource + statements: + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "identity.source")' + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "user.account_uuid") and not IsString(attributes["user.account_uuid"])' + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "user.email") and not IsString(attributes["user.email"])' + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "user.id") and not IsString(attributes["user.id"])' + - 'set(attributes["identity.source"], "unverified") where IsString(attributes["user.account_uuid"]) and IsMatch(attributes["user.account_uuid"], "^[[:space:]]*$")' + - 'set(attributes["identity.source"], "unverified") where IsString(attributes["user.email"]) and IsMatch(attributes["user.email"], "^[[:space:]]*$")' + - 'set(attributes["identity.source"], "unverified") where IsString(attributes["user.id"]) and IsMatch(attributes["user.id"], "^[[:space:]]*$")' + - 'keep_keys(attributes, ["service.version", "identity.source", "user.account_uuid", "user.email", "user.id", "session.id"])' + - 'delete_key(attributes, "service.version") where not IsString(attributes["service.version"])' + - 'delete_key(attributes, "service.version") where IsString(attributes["service.version"]) and not IsMatch(attributes["service.version"], "^[0-9]{1,5}[.][0-9]{1,5}[.][0-9]{1,5}$")' + - 'delete_key(attributes, "user.account_uuid") where not IsString(attributes["user.account_uuid"])' + - 'delete_key(attributes, "user.email") where not IsString(attributes["user.email"])' + - 'delete_key(attributes, "user.id") where not IsString(attributes["user.id"])' + - 'delete_key(attributes, "user.account_uuid") where IsString(attributes["user.account_uuid"]) and IsMatch(attributes["user.account_uuid"], "^[[:space:]]*$")' + - 'delete_key(attributes, "user.email") where IsString(attributes["user.email"]) and IsMatch(attributes["user.email"], "^[[:space:]]*$")' + - 'delete_key(attributes, "user.id") where IsString(attributes["user.id"]) and IsMatch(attributes["user.id"], "^[[:space:]]*$")' + - 'delete_key(attributes, "session.id") where not IsString(attributes["session.id"])' + - 'set(attributes["service.name"], "claude-code")' + - 'set(attributes["fleetdiff.capture.profile"], "claude-code-logs/v1")' + - context: scope + statements: + - 'keep_keys(attributes, [])' + - 'set(name, "fleetdiff.claude-code-capture")' + - 'set(version, "1")' + - context: log + statements: + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "identity.source")' + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "user.account_uuid") and not IsString(attributes["user.account_uuid"])' + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "user.email") and not IsString(attributes["user.email"])' + - 'set(attributes["identity.source"], "unverified") where ContainsValue(Keys(attributes), "user.id") and not IsString(attributes["user.id"])' + - 'set(attributes["identity.source"], "unverified") where IsString(attributes["user.account_uuid"]) and IsMatch(attributes["user.account_uuid"], "^[[:space:]]*$")' + - 'set(attributes["identity.source"], "unverified") where IsString(attributes["user.email"]) and IsMatch(attributes["user.email"], "^[[:space:]]*$")' + - 'set(attributes["identity.source"], "unverified") where IsString(attributes["user.id"]) and IsMatch(attributes["user.id"], "^[[:space:]]*$")' + - 'keep_keys(attributes, ["event.name", "event.sequence", "input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens", "cost_usd", "cost_usd_micros", "identity.source", "user.account_uuid", "user.email", "user.id", "session.id", "prompt.id", "status_code", "attempt", "total_attempts"])' + - 'delete_key(attributes, "event.sequence") where not IsInt(attributes["event.sequence"])' + - 'delete_key(attributes, "input_tokens") where not IsInt(attributes["input_tokens"])' + - 'delete_key(attributes, "output_tokens") where not IsInt(attributes["output_tokens"])' + - 'delete_key(attributes, "cache_read_tokens") where not IsInt(attributes["cache_read_tokens"])' + - 'delete_key(attributes, "cache_creation_tokens") where not IsInt(attributes["cache_creation_tokens"])' + - 'delete_key(attributes, "cost_usd") where not IsDouble(attributes["cost_usd"]) and not IsInt(attributes["cost_usd"])' + - 'delete_key(attributes, "cost_usd_micros") where not IsInt(attributes["cost_usd_micros"])' + - 'delete_key(attributes, "status_code") where not IsInt(attributes["status_code"])' + - 'delete_key(attributes, "attempt") where not IsInt(attributes["attempt"])' + - 'delete_key(attributes, "total_attempts") where not IsInt(attributes["total_attempts"])' + - 'delete_key(attributes, "user.account_uuid") where not IsString(attributes["user.account_uuid"])' + - 'delete_key(attributes, "user.email") where not IsString(attributes["user.email"])' + - 'delete_key(attributes, "user.id") where not IsString(attributes["user.id"])' + - 'delete_key(attributes, "user.account_uuid") where IsString(attributes["user.account_uuid"]) and IsMatch(attributes["user.account_uuid"], "^[[:space:]]*$")' + - 'delete_key(attributes, "user.email") where IsString(attributes["user.email"]) and IsMatch(attributes["user.email"], "^[[:space:]]*$")' + - 'delete_key(attributes, "user.id") where IsString(attributes["user.id"]) and IsMatch(attributes["user.id"], "^[[:space:]]*$")' + - 'delete_key(attributes, "session.id") where not IsString(attributes["session.id"])' + - 'delete_key(attributes, "prompt.id") where not IsString(attributes["prompt.id"])' + - 'set(attributes["event.name"], "api_request") where body == "claude_code.api_request"' + - 'set(attributes["event.name"], "api_error") where body == "claude_code.api_error"' + - 'set(attributes["event.name"], "api_retries_exhausted") where body == "claude_code.api_retries_exhausted"' + - 'set(event_name, body)' + - 'set(severity_text, "")' + - 'set(trace_id, TraceID(0x00000000000000000000000000000000))' + - 'set(span_id, SpanID(0x0000000000000000))' + # At this pin compaction rebuilds envelopes without their untrusted schema URLs. + # The integration test poisons both URLs; rerun before changing the image. + groupbyattrs/safe_envelope: + keys: [] +exporters: + file/capture: + path: /capture/logs.jsonl + format: json + flush_interval: 100ms +service: + telemetry: + logs: + level: error + metrics: + level: none + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, filter/events, transform/privacy, groupbyattrs/safe_envelope] + exporters: [file/capture] diff --git a/examples/claude-code-logs/fixtures.py b/examples/claude-code-logs/fixtures.py new file mode 100644 index 0000000..ca1fc0e --- /dev/null +++ b/examples/claude-code-logs/fixtures.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +# Code authors: Vijay and Codex +"""Fresh deterministic OTLP examples, never copied from native captures.""" +import argparse +import copy +from datetime import datetime, timezone +import json +import os +from pathlib import Path + +SENTINEL = "SYNTHETIC_PRIVATE_CONTENT_DO_NOT_CAPTURE" +IDENTITY = {"user.account_uuid": "synthetic-account-a", "user.email": "account-a@example.invalid", + "user.id": "synthetic-installation-a", "session.id": "synthetic-session-a"} + + +def value(item): + if isinstance(item, str): + return {"stringValue": item} + if isinstance(item, int): + return {"intValue": str(item)} + if isinstance(item, float): + return {"doubleValue": item} + return {"kvlistValue": {"values": attributes(item)}} + + +def attributes(items): + return [{"key": key, "value": value(item)} for key, item in items.items()] + + +def record(event="api_request", sequence=5, day=11): + fields = {"event.name": event, "event.sequence": sequence, + **IDENTITY, "prompt.id": "synthetic-prompt-a"} + if event == "api_request": + fields.update(input_tokens=7, output_tokens=4, cache_read_tokens=11, + cache_creation_tokens=5, cost_usd=0.00003435, cost_usd_micros=34, + model="synthetic-model-a") + else: + fields.update(status_code=429, **({"attempt": 3} if event == "api_error" else {"total_attempts": 3})) + return {"timeUnixNano": str(int(datetime(2026, 10, day, tzinfo=timezone.utc).timestamp()) * 1000000000), + "body": {"stringValue": "claude_code." + event}, "attributes": attributes(fields)} + + +def envelope(records): + return {"resourceLogs": [{"resource": {"attributes": attributes({ + "service.name": "claude-code", "service.version": "2.1.296", + "fleetdiff.capture.profile": "claude-code-logs/v1", **IDENTITY})}, + "scopeLogs": [{"scope": {"name": "synthetic-generator"}, "logRecords": records}]}]} + + +def clean(): + return envelope([record(day=11), record(sequence=8, day=12), + record("api_error", 9, 12), record("api_retries_exhausted", 10, 12)]) + + +def poisoned(): + request = clean() + resource = request["resourceLogs"][0] + resource["schemaUrl"] = "https://example.invalid/" + SENTINEL + resource["resource"]["attributes"] += attributes({ + "process.command_line": SENTINEL, "unknown.resource": {"nested": SENTINEL}}) + resource["resource"]["attributes"][0]["value"] = value(SENTINEL) + resource["resource"]["attributes"][2]["value"] = value(SENTINEL) + scope = resource["scopeLogs"][0] + scope["schemaUrl"] = "https://example.invalid/scope/" + SENTINEL + scope["scope"] = {"name": SENTINEL, "version": SENTINEL, + "attributes": attributes({"unknown.scope": SENTINEL})} + for log in scope["logRecords"]: + log["eventName"] = SENTINEL + log["severityText"] = SENTINEL + log["traceId"] = "1234567890abcdef1234567890abcdef" + log["spanId"] = "1234567890abcdef" + log["attributes"] = [item for item in log["attributes"] if item["key"] != "model"] + log["attributes"] += attributes({"error": SENTINEL, "prompt": SENTINEL, + "model": SENTINEL, "query_source": SENTINEL, + "unknown.attr": {"code": SENTINEL}}) + for body in (value(SENTINEL), value({"code": SENTINEL}), {"arrayValue": {"values": [value(SENTINEL)]}}): + unknown = record() + unknown["body"] = body + scope["logRecords"].append(unknown) + unknown = record("user_prompt") + unknown["attributes"] += attributes({"prompt": SENTINEL}) + scope["logRecords"].append(unknown) + mismatch = record() + mismatch["body"] = value("claude_code.api_error") + scope["logRecords"].append(mismatch) + return request + + +def malformed(): + request = envelope([record()]) + resource = request["resourceLogs"][0] + resource["resource"]["attributes"] = attributes({"service.version": SENTINEL, "user.email": {"code": SENTINEL}}) + log = resource["scopeLogs"][0]["logRecords"][0] + log["attributes"] = attributes({"event.name": "api_request", "input_tokens": SENTINEL, + "output_tokens": {"code": SENTINEL}, "cost_usd": SENTINEL, "cost_usd_micros": SENTINEL, + "event.sequence": SENTINEL, "user.id": {"code": SENTINEL}}) + return request + + +def conflict(): + request = copy.deepcopy(clean()) + resource = request["resourceLogs"][0] + resource["resource"]["attributes"] += attributes({"irrelevant": SENTINEL}) + for item in resource["resource"]["attributes"]: + if item["key"] == "user.account_uuid": + item["value"] = value("synthetic-conflicting-owner") + return request + + +def unverified_identity_cases(): + cases = [] + for location in ("resource", "log"): + for key in ("identity.source", "user.account_uuid", "user.email", "user.id"): + for original in ({}, value(42), value({"nested": SENTINEL}), value(""), value(" \t")): + sequence = 100 + len(cases) + request = envelope([record(sequence=sequence)]) + resource = request["resourceLogs"][0] + target = resource["resource"] if location == "resource" else resource["scopeLogs"][0]["logRecords"][0] + target["attributes"] = [item for item in target["attributes"] if item["key"] != key] + target["attributes"].append({"key": key, "value": original}) + cases.append((sequence, location, key, request)) + request = envelope([record(sequence=100 + len(cases))]) + resource = request["resourceLogs"][0] + target = resource["resource"] if location == "resource" else resource["scopeLogs"][0]["logRecords"][0] + target["attributes"] += attributes({"identity.source": SENTINEL}) + cases.append((100 + len(cases), location, "identity.source", request)) + return cases + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--variant", choices=("clean", "poisoned", "malformed", "conflict"), default="clean") + parser.add_argument("--output", type=Path, help="new synthetic file; otherwise print JSON") + args = parser.parse_args() + encoded = json.dumps(globals()[args.variant](), separators=(",", ":")) + "\n" + if args.output: + with args.output.open("x", encoding="ascii") as output: + os.fchmod(output.fileno(), 0o600) + output.write(encoded) + else: + print(encoded, end="") diff --git a/examples/claude-code-logs/test_capture.py b/examples/claude-code-logs/test_capture.py new file mode 100644 index 0000000..0799e1c --- /dev/null +++ b/examples/claude-code-logs/test_capture.py @@ -0,0 +1,370 @@ +# SPDX-License-Identifier: Apache-2.0 +# Code authors: Vijay and Codex +"""Capture-only conformance. No client, importer, credentials or paid endpoints.""" +import hashlib +import http.client +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest import mock +import urllib.error + +from capture import Capture, IMAGE, MAX_REQUEST, RELAY_IMAGE, ROOT, docker, send +import fixtures + +DOCKER = os.environ.get("FLEETDIFF_CLAUDE_CAPTURE_DOCKER") == "1" +CLI = os.environ.get("FLEETDIFF_CLAUDE_CAPTURE_CLI") == "1" +EVIDENCE = None + + +def evidence_root(): + global EVIDENCE + if EVIDENCE is None: + requested = os.environ.get("FLEETDIFF_CAPTURE_EVIDENCE") + if requested: + EVIDENCE = Path(requested).resolve() + if ROOT.parents[1] == EVIDENCE or ROOT.parents[1] in EVIDENCE.parents: + raise ValueError("test evidence must be outside the repository") + EVIDENCE.mkdir(mode=0o700) + else: + EVIDENCE = Path(tempfile.mkdtemp(prefix="fleetdiff-claude-capture-")) + EVIDENCE.chmod(0o700) + return EVIDENCE + + +def private_json(path, data): + with path.open("x", encoding="ascii") as output: + os.fchmod(output.fileno(), 0o600) + json.dump(data, output, indent=2) + output.write("\n") + + +def records(path): + return [(resource, scope, log) + for line in path.read_bytes().splitlines() + for resource in json.loads(line).get("resourceLogs", []) + for scope in resource.get("scopeLogs", []) + for log in scope.get("logRecords", [])] + + +def attrs(items): + return {item["key"]: item["value"] for item in items} + + +def investigate(path, group="user", output_format="json", deduplicate=False): + environment = dict(os.environ) + binary = environment.get("FLEETDIFF_BIN") + if binary: + command = [str(Path(binary).resolve())] + else: + environment.setdefault("GOTOOLCHAIN", "local") + environment.setdefault("GOPROXY", "off") + environment.setdefault("GOCACHE", str(evidence_root() / "go-cache")) + command = ["go", "run", "./cmd/fleetdiff"] + command += ["investigate", "--claude-code-logs", str(path), + "--before-period", "2026-10-11T00:00:00Z/2026-10-12T00:00:00Z", + "--after-period", "2026-10-12T00:00:00Z/2026-10-13T00:00:00Z", + "--group-by", group, "--format", output_format] + if deduplicate: + command.append("--deduplicate-exact") + return subprocess.run(command, cwd=ROOT.parents[1], env=environment, + capture_output=True, text=True, timeout=120) + + +class FixtureTests(unittest.TestCase): + def test_cli_uses_supplied_binary_and_preserves_ci_environment(self): + with mock.patch.dict(os.environ, {"FLEETDIFF_BIN": "/ci/bin/fleetdiff", "GOCACHE": "/ci/cache", + "GOTOOLCHAIN": "local", "GOSUMDB": "sum.golang.org"}), \ + mock.patch("subprocess.run") as run: + investigate(Path("fixture.jsonl")) + self.assertEqual(run.call_args.args[0][0], "/ci/bin/fleetdiff") + self.assertEqual(run.call_args.kwargs["env"]["GOCACHE"], "/ci/cache") + self.assertEqual(run.call_args.kwargs["env"]["GOTOOLCHAIN"], "local") + self.assertEqual(run.call_args.kwargs["env"]["GOSUMDB"], "sum.golang.org") + + def test_deterministic_fresh_fixture_and_profile(self): + self.assertEqual(fixtures.clean(), fixtures.clean()) + batch = fixtures.clean()["resourceLogs"][0] + self.assertEqual(attrs(batch["resource"]["attributes"])["fleetdiff.capture.profile"], + {"stringValue": "claude-code-logs/v1"}) + request = batch["scopeLogs"][0]["logRecords"][0] + self.assertEqual(request["timeUnixNano"], "1791676800000000000") + self.assertEqual(attrs(request["attributes"])["cost_usd"], {"doubleValue": 0.00003435}) + self.assertEqual(attrs(request["attributes"])["model"], {"stringValue": "synthetic-model-a"}) + + def test_existing_directory_and_symlink_are_rejected(self): + with tempfile.TemporaryDirectory() as temporary, mock.patch("capture.docker", return_value=""): + path = Path(temporary) + with self.assertRaises(FileExistsError): + Capture(path).start() + link = path / "link" + link.symlink_to(path, target_is_directory=True) + with self.assertRaises(FileExistsError): + Capture(link).start() + + def test_cli_rejects_invalid_duration_without_docker(self): + result = subprocess.run([sys.executable, "-B", str(ROOT / "capture.py"), + "/unused", "--seconds", "0"], capture_output=True, timeout=10) + self.assertEqual(result.returncode, 2) + + def test_partial_acceptance_is_failure(self): + response = mock.MagicMock() + response.__enter__.return_value.read.return_value = b'{"partialSuccess":{"rejectedLogRecords":"1"}}' + with mock.patch("urllib.request.build_opener") as opener: + opener.return_value.open.return_value = response + with self.assertRaisesRegex(RuntimeError, "partial acceptance"): + send("http://127.0.0.1:1/v1/logs", b"{}") + + def test_relay_rejects_unbounded_encoded_and_nonlog_requests(self): + session = Capture("/unused") + session.start_relay() + try: + with mock.patch("capture.subprocess.run") as execute: + for path, headers in [ + ("/v1/traces", {"Content-Length": "0"}), + ("/v1/logs", {"Content-Length": str(MAX_REQUEST + 1)}), + ("/v1/logs", {"Content-Length": "0", "Content-Encoding": "gzip"}), + ("/v1/logs", {"Content-Length": "0", "Content-Type": "application/x-protobuf"}), + ("/v1/logs", {"Content-Length": "0", "Transfer-Encoding": "chunked"}), + ]: + connection = http.client.HTTPConnection(*session.server.server_address, timeout=3) + connection.request("POST", path, headers={"Content-Type": "application/json", **headers}) + self.assertEqual(connection.getresponse().status, 413) + connection.close() + execute.assert_not_called() + finally: + session.stop() + + +@unittest.skipUnless(CLI, "set FLEETDIFF_CLAUDE_CAPTURE_CLI=1 for the local-checkout smoke check") +class CLIExampleTests(unittest.TestCase): + def test_generated_model_fixture_user_session_and_text(self): + path = evidence_root() / "cli-fixture.jsonl" + generated = subprocess.run([sys.executable, "-B", str(ROOT / "fixtures.py"), "--output", str(path)], + capture_output=True, text=True, timeout=10) + self.assertEqual(generated.returncode, 0, generated.stderr) + for group in ("user", "session"): + result = investigate(path, group=group) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["unit"], "observed api_request records") + self.assertEqual(report["finality"], "unknown") + self.assertTrue(report["models"]) + for period in ("before", "after"): + self.assertEqual(report[period]["observed_api_request_records"], 1) + self.assertEqual(report[period]["inclusive_input"]["tokens"], 23) + self.assertEqual(report[period]["output"]["tokens"], 4) + self.assertEqual(report[period]["recorded_cost_usd"], "0.00003435") + self.assertEqual(report["after"]["api_error_events"], 1) + self.assertEqual(report["after"]["api_retries_exhausted_events"], 1) + private_json(evidence_root() / ("cli-" + group + ".json"), report) + result = investigate(path, output_format="text") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("unknown", result.stdout) + + +@unittest.skipUnless(DOCKER, "set FLEETDIFF_CLAUDE_CAPTURE_DOCKER=1; pinned image must already be local") +class CollectorTests(unittest.TestCase): + def capture(self, name, batches, max_bytes=8 * 1024 * 1024): + root = evidence_root() / name + root.mkdir(mode=0o700) + private_json(root / "input.json", batches) + session = Capture(root / "capture", max_bytes=max_bytes) + try: + endpoint = session.start() + network = json.loads(docker("network", "inspect", session.name))[0] + container = json.loads(docker("inspect", session.name))[0] + self.assertTrue(network["Internal"]) + self.assertFalse(container["HostConfig"]["PortBindings"]) + self.assertEqual(session.server.server_address[0], "127.0.0.1") + relay = json.loads(docker("inspect", session.name + "-relay"))[0] + self.assertEqual(relay["HostConfig"]["NetworkMode"], "container:" + container["Id"]) + self.assertEqual(container["HostConfig"]["LogConfig"]["Type"], "none") + self.assertTrue(container["HostConfig"]["ReadonlyRootfs"]) + self.assertEqual(container["Config"]["Image"], IMAGE) + self.assertIn({"Name": "fsize", "Hard": max_bytes, "Soft": max_bytes}, container["HostConfig"]["Ulimits"]) + for batch in batches: + send(endpoint, json.dumps(batch).encode()) + finally: + session.stop() + metadata = session.finalize("test-stopped") + self.assertTrue(metadata["private_modes"]) + private_json(root / "result.json", {"passed": True, "capture": metadata}) + return session.directory / "logs.jsonl" + + def test_allowlists_preserve_types_ids_and_duplicates(self): + batch = fixtures.poisoned() + path = self.capture("allowlists", [batch, batch]) + data = path.read_bytes() + self.assertNotIn(fixtures.SENTINEL.encode(), data) + observed = records(path) + self.assertEqual(len(observed), 8) + fingerprints = [hashlib.sha256(json.dumps(log, sort_keys=True).encode()).hexdigest() + for _, _, log in observed] + self.assertEqual(len(set(fingerprints)), 4) + for resource, scope, log in observed: + resource_attrs = attrs(resource["resource"]["attributes"]) + self.assertEqual(set(resource_attrs), {"service.name", "service.version", "fleetdiff.capture.profile", *fixtures.IDENTITY}) + self.assertEqual(resource_attrs["service.name"], {"stringValue": "claude-code"}) + self.assertEqual(resource_attrs["fleetdiff.capture.profile"], {"stringValue": "claude-code-logs/v1"}) + self.assertNotIn("schemaUrl", resource) + self.assertNotIn("schemaUrl", scope) + self.assertEqual(scope["scope"].get("attributes", []), []) + self.assertEqual(scope["scope"]["name"], "fleetdiff.claude-code-capture") + fields = attrs(log["attributes"]) + for key, item in fixtures.IDENTITY.items(): + self.assertEqual(fields[key], {"stringValue": item}) + self.assertNotIn("error", fields) + self.assertNotIn("model", fields) + self.assertNotIn("query_source", fields) + self.assertFalse(log.get("severityText")) + self.assertFalse(log.get("traceId")) + self.assertFalse(log.get("spanId")) + self.assertEqual(log["body"]["stringValue"], "claude_code." + fields["event.name"]["stringValue"]) + if fields["event.name"]["stringValue"] == "api_request": + for key, number in {"input_tokens": 7, "output_tokens": 4, "cache_read_tokens": 11, + "cache_creation_tokens": 5, "cost_usd_micros": 34}.items(): + self.assertEqual(fields[key], {"intValue": str(number)}) + self.assertEqual(fields["cost_usd"], {"doubleValue": 0.00003435}) + else: + self.assertNotIn("input_tokens", fields) + self.assertNotIn("cost_usd", fields) + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) + self.assertEqual(stat.S_IMODE(path.parent.stat().st_mode), 0o700) + + def test_wrong_type_content_is_removed_not_coerced(self): + path = self.capture("types", [fixtures.malformed()]) + self.assertNotIn(fixtures.SENTINEL.encode(), path.read_bytes()) + resource, _, log = records(path)[0] + self.assertEqual(set(attrs(resource["resource"]["attributes"])), {"service.name", "fleetdiff.capture.profile", "identity.source"}) + self.assertEqual(attrs(log["attributes"]), {"event.name": {"stringValue": "api_request"}, + "identity.source": {"stringValue": "unverified"}}) + + def test_identity_regimes_and_malformed_owners_cannot_fall_back(self): + cases = fixtures.unverified_identity_cases() + path = self.capture("identity-regimes", [case[3] for case in cases]) + self.assertNotIn(fixtures.SENTINEL.encode(), path.read_bytes()) + expected = {sequence: (location, key) for sequence, location, key, _ in cases} + self.assertEqual(len(records(path)), len(cases)) + for resource, _, log in records(path): + fields = attrs(log["attributes"]) + sequence = int(fields["event.sequence"]["intValue"]) + location, malformed_key = expected[sequence] + target = attrs(resource["resource"]["attributes"]) if location == "resource" else fields + self.assertEqual(target["identity.source"], {"stringValue": "unverified"}) + if malformed_key != "identity.source": + self.assertNotIn(malformed_key, target) + if CLI: + result = investigate(path) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["before"]["unattributed_records"], len(cases)) + private_json(evidence_root() / "cli-identity-regimes.json", report) + + def test_conflicting_identity_locality_is_not_reconciled(self): + path = self.capture("conflict", [fixtures.conflict()]) + for resource, _, log in records(path): + self.assertEqual(attrs(resource["resource"]["attributes"])["user.account_uuid"], + {"stringValue": "synthetic-conflicting-owner"}) + self.assertEqual(attrs(log["attributes"])["user.account_uuid"], + {"stringValue": "synthetic-account-a"}) + + def test_unknown_events_and_bodies_are_dropped(self): + batch = fixtures.poisoned() + scope = batch["resourceLogs"][0]["scopeLogs"][0] + scope["logRecords"] = scope["logRecords"][4:] + path = self.capture("unknown", [batch]) + self.assertEqual(records(path), []) + self.assertNotIn(fixtures.SENTINEL.encode(), path.read_bytes()) + + def test_zero_negative_and_missing_remain_distinct(self): + batch = fixtures.envelope([fixtures.record()]) + log = batch["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + log["attributes"] = fixtures.attributes({"event.name": "api_request", "input_tokens": 0, + "output_tokens": -1, "cost_usd": 0}) + path = self.capture("missing", [batch]) + fields = attrs(records(path)[0][2]["attributes"]) + self.assertEqual(fields["input_tokens"], {"intValue": "0"}) + self.assertEqual(fields["output_tokens"], {"intValue": "-1"}) + self.assertNotIn("cache_read_tokens", fields) + self.assertEqual(fields["cost_usd"], {"intValue": "0"}) + + def test_hard_file_cap_fails_closed(self): + session = Capture(evidence_root() / "size-cap", max_bytes=1024) + try: + endpoint = session.start() + try: + send(endpoint, json.dumps(fixtures.clean()).encode()) + except (urllib.error.URLError, ConnectionError, TimeoutError): + pass + finally: + session.stop() + self.assertLessEqual((session.directory / "logs.jsonl").stat().st_size, 1024) + with self.assertRaises(RuntimeError): + session.finalize("size-cap") + + def test_cli_timer_flushes_and_cleans_up(self): + root = evidence_root() / "cli" + process = subprocess.Popen([sys.executable, "-B", str(ROOT / "capture.py"), str(root), "--seconds", "2"], + cwd=evidence_root(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + first = process.stdout.readline().strip() + self.assertTrue(first.startswith("OTLP logs endpoint: http://127.0.0.1:")) + send(first.split(": ", 1)[1], json.dumps(fixtures.clean()).encode()) + output, error = process.communicate(timeout=30) + self.assertEqual(process.returncode, 0, error) + self.assertNotIn(fixtures.SENTINEL, output + error) + self.assertEqual(len(records(root / "logs.jsonl")), 4) + self.assertEqual(json.loads((root / "capture.json").read_text())["stop_reason"], "timer") + finally: + if process.poll() is None: + process.terminate() + process.communicate(timeout=30) + + def test_cli_termination_flushes_and_cleans_up(self): + root = evidence_root() / "signal" + process = subprocess.Popen([sys.executable, "-B", str(ROOT / "capture.py"), str(root), "--seconds", "30"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + endpoint = process.stdout.readline().strip().split(": ", 1)[1] + send(endpoint, json.dumps(fixtures.clean()).encode()) + process.terminate() + _, error = process.communicate(timeout=30) + self.assertEqual(process.returncode, 0, error) + self.assertEqual(len(records(root / "logs.jsonl")), 4) + self.assertEqual(json.loads((root / "capture.json").read_text())["stop_reason"], "signal") + finally: + if process.poll() is None: + process.terminate() + process.communicate(timeout=30) + + @unittest.skipUnless(CLI, "local-checkout CLI smoke check not enabled") + def test_filtered_file_is_accepted_and_exact_copies_are_optional(self): + batch = fixtures.poisoned() + path = self.capture("cli-filtered", [batch, batch]) + for deduplicate, count in ((False, 2), (True, 1)): + result = investigate(path, deduplicate=deduplicate) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + for period in ("before", "after"): + self.assertEqual(report[period]["observed_api_request_records"], count) + self.assertEqual(report[period]["inclusive_input"]["tokens"], 23 * count) + self.assertEqual(report[period]["output"]["tokens"], 4 * count) + self.assertEqual(report["finality"], "unknown") + private_json(evidence_root() / ("cli-filtered-dedup-" + str(deduplicate) + ".json"), report) + + +if __name__ == "__main__": + result = unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__])) + if EVIDENCE is not None: + private_json(EVIDENCE / "summary.json", {"tests": result.testsRun, "failures": len(result.failures), + "errors": len(result.errors), "skipped": len(result.skipped), "image": IMAGE, + "relay_image": RELAY_IMAGE, + "config_sha256": hashlib.sha256((ROOT / "collector.yaml").read_bytes()).hexdigest()}) + print("Private synthetic evidence:", EVIDENCE) + sys.exit(not result.wasSuccessful()) diff --git a/examples/claude-code-logs/test_rotation.py b/examples/claude-code-logs/test_rotation.py new file mode 100644 index 0000000..fb99ebd --- /dev/null +++ b/examples/claude-code-logs/test_rotation.py @@ -0,0 +1,286 @@ +# SPDX-License-Identifier: Apache-2.0 +# Code authors: Vijay and Codex +"""Pinned rotation/retention checks using synthetic data and cached images only.""" +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import time +import unittest +from unittest import mock + +import capture +import fixtures +from test_capture import attrs, evidence_root, private_json, records + +ROOT = capture.ROOT +OVERLAY = ROOT / "collector-long-running.json" +DOCKER = os.environ.get("FLEETDIFF_CLAUDE_CAPTURE_DOCKER") == "1" +CLI = os.environ.get("FLEETDIFF_CLAUDE_CAPTURE_CLI") == "1" + + +def policy(): + return json.loads(OVERLAY.read_text(encoding="ascii")) + + +def overlay_docker(*args): + # Reuse the bounded helper's isolation/relay without changing its behavior. + args = list(args) + if args[0] == "create" and "--config=/etc/capture.yaml" in args: + index = args.index(capture.IMAGE) + args[index:index] = ["--mount", f"type=bind,src={OVERLAY},dst=/etc/rotation.json,readonly", + "--env", "CLAUDE_CAPTURE_DIR=/capture"] + args.append("--config=/etc/rotation.json") + return REAL_DOCKER(*args) + + +REAL_DOCKER = capture.docker + + +def import_files(paths, *, deduplicate=False): + binary = os.environ.get("FLEETDIFF_BIN") + if not binary: + raise ValueError("rotation CLI checks require FLEETDIFF_BIN") + command = [str(Path(binary).resolve()), "investigate"] + for path in paths: + command += ["--claude-code-logs", str(path)] + command += ["--group-by", "user", "--format", "json"] + if deduplicate: + command.append("--deduplicate-exact") + return subprocess.run(command, capture_output=True, text=True, timeout=30) + + +def write_batch(path, batch): + with path.open("x", encoding="ascii") as output: + os.fchmod(output.fileno(), 0o600) + json.dump(batch, output, separators=(",", ":")) + output.write("\n") + + +class PolicyTests(unittest.TestCase): + def test_overlay_reuses_redaction_and_only_changes_route_and_exporter(self): + value = policy() + self.assertEqual(set(value), {"receivers", "exporters"}) + self.assertEqual(value["receivers"]["otlp"]["protocols"]["http"], + {"endpoint": "127.0.0.1:4318"}) + exporter = value["exporters"]["file/capture"] + self.assertEqual(exporter["rotation"], {"max_megabytes": 8, "max_days": 15, + "max_backups": 15, "localtime": False}) + self.assertFalse(exporter["append"]) + self.assertFalse(exporter["create_directory"]) + self.assertNotIn("group_by", exporter) + self.assertNotIn("compression", exporter) + self.assertEqual(exporter["path"], "${env:CLAUDE_CAPTURE_DIR}/logs.jsonl") + + +@unittest.skipUnless(DOCKER, "set FLEETDIFF_CLAUDE_CAPTURE_DOCKER=1; cached images required") +class RotationTests(unittest.TestCase): + def validate(self, overlay): + return subprocess.run([ + "docker", "run", "--rm", "--pull", "never", "--network", "none", + "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--env", "CLAUDE_CAPTURE_DIR=/capture", + "--mount", f"type=bind,src={ROOT / 'collector.yaml'},dst=/etc/base.yaml,readonly", + "--mount", f"type=bind,src={overlay},dst=/etc/rotation.json,readonly", + capture.IMAGE, "validate", "--config=/etc/base.yaml", "--config=/etc/rotation.json", + ], capture_output=True, text=True, timeout=30) + + def test_pinned_binary_accepts_composition_and_rejects_append_rotation(self): + capture.docker("image", "inspect", capture.IMAGE) + valid = self.validate(OVERLAY) + self.assertEqual(valid.returncode, 0, valid.stderr) + invalid = policy() + invalid["exporters"]["file/capture"]["append"] = True + path = evidence_root() / "invalid-append.json" + private_json(path, invalid) + rejected = self.validate(path) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("append and rotation", rejected.stderr) + private_json(evidence_root() / "validation.json", { + "valid_exit": valid.returncode, "append_rotation_exit": rejected.returncode, + "image": capture.IMAGE, + }) + + def test_exact_policy_rotation_age_count_privacy_and_modes(self): + root = evidence_root() / "rotation" + root.mkdir(mode=0o700) + session = capture.Capture(root / "capture", max_bytes=32 * 1024 * 1024) + count, initial = 0, set() + now = datetime.now(timezone.utc) + old_name = "" + try: + with mock.patch("capture.docker", side_effect=overlay_docker): + endpoint = session.start() + self.assertTrue(json.loads(capture.docker("network", "inspect", session.name))[0]["Internal"]) + self.assertFalse(json.loads(capture.docker("inspect", session.name))[0]["HostConfig"]["PortBindings"]) + # Empty synthetic backups exercise the actual filename-age policy, + # not a fake clock or a 15-day wall-clock soak. + for i in range(2): + timestamp = now - (timedelta(days=16) if i == 1 else timedelta(minutes=1)) + name = "logs-" + timestamp.strftime("%Y-%m-%dT%H-%M-%S.%f")[:-3] + "-size.jsonl" + with (session.directory / name).open("xb") as output: + os.fchmod(output.fileno(), 0o600) + initial.add(name) + if i == 1: + old_name = name + unrelated = session.directory / "unrelated.keep" + with unrelated.open("xb") as output: + os.fchmod(output.fileno(), 0o600) + + batch = fixtures.poisoned() + batch["resourceLogs"][0]["scopeLogs"][0]["logRecords"] *= 64 + payload = json.dumps(batch).encode() + self.assertLess(len(payload), capture.MAX_REQUEST) + expected_per_batch = 4 * 64 + capture.send(endpoint, payload) + count += 1 + deadline = time.monotonic() + 5 + while (session.directory / old_name).exists(): + if time.monotonic() >= deadline: + self.fail("age cleanup below the backup-count limit did not settle") + time.sleep(0.05) + self.assertTrue(all((session.directory / name).exists() + for name in initial if name != old_name)) + # Now independently exceed max_backups with recent empty files. + for i in range(17): + timestamp = now - timedelta(minutes=i + 2) + name = "logs-" + timestamp.strftime("%Y-%m-%dT%H-%M-%S.%f")[:-3] + "-size.jsonl" + with (session.directory / name).open("xb") as output: + os.fchmod(output.fileno(), 0o600) + initial.add(name) + for _ in range(256): + capture.send(endpoint, payload) + count += 1 + current = {path.name for path in session.directory.glob("logs-*.jsonl")} + if len(current - initial) >= 2: + break + else: + self.fail("bounded synthetic workload did not trigger two rotations") + + deadline = time.monotonic() + 5 + while (len(list(session.directory.glob("logs-*.jsonl"))) > 15 + or (session.directory / old_name).exists()): + if time.monotonic() >= deadline: + self.fail("asynchronous retention cleanup did not settle") + time.sleep(0.05) + finally: + session.stop() + + backups = sorted(session.directory.glob("logs-*.jsonl")) + self.assertEqual(len(backups), 15) + self.assertFalse((session.directory / old_name).exists()) + self.assertTrue(unrelated.exists(), "exporter must not delete unrelated paths") + self.assertGreaterEqual(len({path.name for path in backups} - initial), 2) + outputs = [*backups, session.directory / "logs.jsonl"] + observed, modes, sizes = 0, {}, {} + for path in outputs: + data = path.read_bytes() + self.assertNotIn(fixtures.SENTINEL.encode(), data) + sizes[path.name] = len(data) + self.assertLessEqual(len(data), 8 * 1024 * 1024) + modes[path.name] = oct(stat.S_IMODE(path.stat().st_mode)) + for resource, scope, log in records(path): + observed += 1 + self.assertNotIn("schemaUrl", resource) + self.assertNotIn("schemaUrl", scope) + self.assertEqual(scope["scope"].get("attributes", []), []) + self.assertEqual(attrs(resource["resource"]["attributes"])["fleetdiff.capture.profile"], + {"stringValue": "claude-code-logs/v1"}) + self.assertNotIn("model", attrs(log["attributes"])) + self.assertNotIn("error", attrs(log["attributes"])) + self.assertEqual(observed, count * expected_per_batch) + self.assertEqual(stat.S_IMODE(session.directory.stat().st_mode), 0o700) + # This pin forces 0644 on new rotated files. Do not certify a 0600 policy. + self.assertEqual(modes["logs.jsonl"], "0o644") + self.assertIn("0o644", [modes[path.name] for path in backups]) + private_json(root / "result.json", { + "passed": True, "batches_sent": count, "records": observed, + "backups_after_cleanup": len(backups), "age_seed_removed": old_name, + "age_check_seed_count": 2, "recent_seed_retained_during_age_check": True, + "unrelated_file_retained": True, "directory_mode": "0o700", + "file_modes": modes, "file_bytes": sizes, "strict_0600_satisfied": False, + "active_file_age_ttl": "not provided by exporter", + "source_finality": "unknown", "image": capture.IMAGE, + }) + if CLI: + paths = [path for path in outputs if path.stat().st_size] + for deduplicate in (False, True): + result = import_files(paths, deduplicate=deduplicate) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn(fixtures.SENTINEL, result.stdout + result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["input_files"], len(paths)) + self.assertEqual(report["period_selection"], "observed_span_halves") + self.assertEqual(report["finality"], "unknown") + self.assertFalse(report["models"]) + expected = 1 if deduplicate else count * 64 + for period in ("before", "after"): + self.assertEqual(report[period]["observed_api_request_records"], expected) + self.assertEqual(report[period]["inclusive_input"]["tokens"], expected * 23) + self.assertEqual(report[period]["output"]["tokens"], expected * 4) + private_json(root / f"cli-dedup-{deduplicate}.json", report) + + +@unittest.skipUnless(CLI, "set FLEETDIFF_CLAUDE_CAPTURE_CLI=1 and FLEETDIFF_BIN") +class DefaultPeriodTests(unittest.TestCase): + def test_repeated_files_retained_or_exact_filtered_and_both_defaults(self): + root = evidence_root() / "defaults" + root.mkdir(mode=0o700) + paths = [root / "first.jsonl", root / "copied.jsonl"] + for path in paths: + write_batch(path, fixtures.clean()) + for deduplicate in (False, True): + result = import_files(paths, deduplicate=deduplicate) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["input_files"], 2) + self.assertEqual(report["period_selection"], "observed_span_halves") + self.assertEqual(report["finality"], "unknown") + for period in ("before", "after"): + self.assertEqual(report[period]["observed_api_request_records"], 1 if deduplicate else 2) + private_json(root / f"halves-dedup-{deduplicate}.json", report) + + weekly_paths = [] + dates = ("2026-09-20", "2026-09-23", "2026-10-02", "2026-10-05") + for i in range(2): + batch = fixtures.clean() + scope = batch["resourceLogs"][0]["scopeLogs"][0] + request = scope["logRecords"][0] + scope["logRecords"] = [] + for date in dates[i * 2:i * 2 + 2]: + log = copy.deepcopy(request) + stamp = datetime.fromisoformat(date).replace(tzinfo=timezone.utc) + log["timeUnixNano"] = str(int(stamp.timestamp()) * 1_000_000_000) + scope["logRecords"].append(log) + path = root / f"weeks-{i}.jsonl" + write_batch(path, batch) + weekly_paths.append(path) + result = import_files(weekly_paths) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["input_files"], 2) + self.assertEqual(report["period_selection"], "complete_utc_weeks") + self.assertEqual(report["outside_periods"], 2) + for period in ("before", "after"): + self.assertEqual(report[period]["observed_api_request_records"], 1) + private_json(root / "complete-weeks.json", report) + + +if __name__ == "__main__": + os.umask(0o077) + suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__]) + result = unittest.TextTestRunner(verbosity=2).run(suite) + private_json(evidence_root() / "rotation-summary.json", { + "tests": result.testsRun, "failures": len(result.failures), + "errors": len(result.errors), "skipped": len(result.skipped), + "image": capture.IMAGE, "relay_image": capture.RELAY_IMAGE, + "base_config_sha256": hashlib.sha256((ROOT / "collector.yaml").read_bytes()).hexdigest(), + "overlay_sha256": hashlib.sha256(OVERLAY.read_bytes()).hexdigest(), + "cli_binary": os.environ.get("FLEETDIFF_BIN") if CLI else None, + }) + sys.exit(not result.wasSuccessful()) diff --git a/internal/clauderecords/accounting.go b/internal/clauderecords/accounting.go new file mode 100644 index 0000000..ced21fb --- /dev/null +++ b/internal/clauderecords/accounting.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "math/big" + "strconv" + "strings" + + "github.com/llm-measurement/fleetdiff/internal/accounting" + commonpb "go.opentelemetry.io/proto/slim/otlp/common/v1" +) + +type observation struct { + input, output, total, read, write *uint64 + cost *big.Rat + costDelta *big.Rat + costKind string + quality map[string]uint64 +} + +// Numeric OTLP values are parsed exactly before the shared token contract sees +// them. An invalid row is excluded; only aggregate overflow aborts a report. +func number(v value) (uint64, bool) { + if v.invalid || len(v.text) > 32 { + return 0, false + } + if v.kind == "doubleValue" { + if strings.ContainsAny(v.text, "xXpP/") { + return 0, false + } + if i := strings.IndexAny(v.text, "eE"); i >= 0 { + e, err := strconv.Atoi(v.text[i+1:]) + if err != nil || e < -400 || e > 400 { + return 0, false + } + } + if _, err := strconv.ParseFloat(v.text, 64); err != nil { + return 0, false + } + n, ok := new(big.Rat).SetString(v.text) + if !ok || n.Sign() < 0 || !n.IsInt() || !n.Num().IsInt64() { + return 0, false + } + return n.Num().Uint64(), true + } + if v.kind != "stringValue" && v.kind != "intValue" { + return 0, false + } + n, err := strconv.ParseUint(v.text, 10, 63) + return n, err == nil +} + +func normalize(r record) observation { + o := observation{quality: map[string]uint64{}} + var parts [4]*uint64 + for i, name := range []string{"input_tokens", "cache_read_tokens", "cache_creation_tokens", "output_tokens"} { + v, present := r.resolve(name) + if !present { + o.quality[name+"_missing"]++ + continue + } + n, ok := number(v) + if !ok { + o.quality[name+"_invalid"]++ + continue + } + parts[i] = &n + if n == 0 { + o.quality["zero_token_origin_unknown"] = 1 + } + } + if parts[0] != nil && parts[1] != nil && parts[2] != nil { + n, err := accounting.AddBounded(*parts[0], *parts[1]) + if err == nil { + n, err = accounting.AddBounded(n, *parts[2]) + } + if err == nil { + o.input = &n + } else { + o.quality["input_sum_invalid"] = 1 + } + } + o.read, o.write, o.output = parts[1], parts[2], parts[3] + attrs := map[string]*commonpb.AnyValue{} + for name, n := range map[string]*uint64{"input_tokens": o.input, "output_tokens": o.output, "cache_read.input_tokens": o.read, "cache_write.input_tokens": o.write} { + if n != nil { + attrs["gen_ai.usage."+name] = &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: int64(*n)}} + } + } + a, err := accounting.Tokens(attrs) + if err != nil { + o.quality["total_sum_invalid"] = 1 + } else if o.input != nil && o.output != nil { + n := a.Metrics[accounting.Prefix+"total_tokens_total"] + o.total = &n + } + v, hasDecimal := r.resolve("cost_usd") + if hasDecimal { + if !v.invalid { + o.cost, _ = accounting.Cost(v.text) + } + if o.cost == nil { + o.quality["cost_invalid"] = 1 + } else { + o.costKind = v.kind + o.quality["cost_decimal"] = 1 + if micros, present := r.resolve("cost_usd_micros"); present { + if n, ok := number(micros); ok { + m := new(big.Rat).SetFrac(new(big.Int).SetUint64(n), big.NewInt(1_000_000)) + o.costDelta = new(big.Rat).Sub(o.cost, m) + } else { + o.quality["cost_micros_invalid"] = 1 + } + } + } + } else if v, ok := r.resolve("cost_usd_micros"); ok { + if n, valid := number(v); valid { + o.cost = new(big.Rat).SetFrac(new(big.Int).SetUint64(n), big.NewInt(1_000_000)) + o.quality["cost_micros_fallback"] = 1 + o.costKind = v.kind + } else { + o.quality["cost_invalid"] = 1 + } + } else { + o.quality["cost_missing"] = 1 + } + if o.cost != nil && o.cost.Sign() == 0 { + o.quality["zero_cost_origin_unknown"] = 1 + } + return o +} diff --git a/internal/clauderecords/identity.go b/internal/clauderecords/identity.go new file mode 100644 index 0000000..e7e2c7c --- /dev/null +++ b/internal/clauderecords/identity.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "encoding/binary" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/llm-measurement/fleetdiff/internal/attribution" + "github.com/llm-measurement/fleetdiff/internal/compare" +) + +type label struct { + alias string + order int + digest [32]byte +} +type catalog struct { + key []byte + scope string + entries map[uint64]label +} + +func owner(r record) (kind, id string) { + if _, present := r.resolve("identity.source"); present { + return "unattributed", "" + } + for _, f := range []struct{ name, kind string }{{"user.account_uuid", "account"}, {"user.email", "email"}, {"user.id", "installation"}} { + if _, present := r.resolve(f.name); present { + v, _ := r.resolve(f.name) + if !v.invalid && v.kind == "stringValue" && strings.TrimSpace(v.text) == "" { + continue + } + if id, ok := r.text(f.name); ok { + return f.kind, id + } + // Conflicting/malformed higher-priority attribution never falls back. + return "unattributed", "" + } + } + return "unattributed", "" +} +func (c *catalog) digest(kind string, parts ...string) ([32]byte, error) { + return attribution.Identity(c.key, "claude-code-logs/v1", kind, append([]string{c.scope}, parts...)...) +} +func (c *catalog) group(r record, group string) (uint64, string, bool, error) { + kind, id := owner(r) + if id == "" { + return 0, kind, false, nil + } + parts := []string{kind, id} + if group == "session" { + session, ok := r.text("session.id") + if !ok { + return 0, kind, false, nil + } + parts = append(parts, session) + } + d, err := c.digest(group, parts...) + if err != nil { + return 0, kind, false, err + } + h := binary.BigEndian.Uint64(d[:8]) + if old, exists := c.entries[h]; exists { + if old.digest != d { + return 0, kind, false, errors.New("identity hash collision; choose a different hashing secret") + } + } else { + if len(c.entries) >= MaxIdentities { + return 0, kind, false, errors.New("capture exceeds 250000 attributed identities; select a smaller capture") + } + prefix := kind + if group == "session" { + prefix = "session" + } + c.entries[h] = label{fmt.Sprintf("%s-%d", prefix, len(c.entries)+1), len(c.entries), d} + } + return h, kind, true, nil +} +func (c *catalog) prompt(r record) ([32]byte, bool, error) { + kind, id := owner(r) + session, hasSession := r.text("session.id") + prompt, hasPrompt := r.text("prompt.id") + if id == "" || !hasSession || !hasPrompt { + return [32]byte{}, false, nil + } + d, err := c.digest("prompt", kind, id, session, prompt) + return d, err == nil, err +} +func (c *catalog) rank(name string, a, b *aggregate, cache bool, opts Options) (compare.Concentration, error) { + x, y := a.top, b.top + if cache { + x, y = a.cacheTop, b.cacheTop + } + r, err := compare.RankCandidates(name, x, y) + if err != nil { + return r, err + } + r.WeightUnit = "reported_tokens_finality_unknown" + key := func(m compare.Mover) uint64 { n, _ := strconv.ParseUint(m.Hash, 16, 64); return n } + abs := func(n int64) int64 { + if n < 0 { + return -n + } + return n + } + slices.SortFunc(r.Movers, func(a, b compare.Mover) int { + x, y := max(abs(a.Delta.Lower), abs(a.Delta.Upper)), max(abs(b.Delta.Lower), abs(b.Delta.Upper)) + if x > y { + return -1 + } + if x < y { + return 1 + } + return c.entries[key(a)].order - c.entries[key(b)].order + }) + r.Movers = r.Movers[:min(opts.Top, len(r.Movers))] + for i := range r.Movers { + m := &r.Movers[i] + m.Item = c.entries[key(*m)].alias + if !opts.ShowHashes { + m.Hash = "" + } + } + return r, nil +} diff --git a/internal/clauderecords/investigate.go b/internal/clauderecords/investigate.go new file mode 100644 index 0000000..4a4a999 --- /dev/null +++ b/internal/clauderecords/investigate.go @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/llm-measurement/fleetdiff/internal/accounting" + "github.com/llm-measurement/fleetdiff/internal/attribution" + "github.com/llm-measurement/fleetdiff/internal/compare" +) + +type Options struct { + BeforePeriod, AfterPeriod, GroupBy, SecretEnv, SourceScope string + Top int + ShowHashes, DeduplicateExact bool + Now time.Time +} + +// Investigate compares log observations, not logical requests or provider sends. +// It never manufactures events for cancellation or infers a final receipt. +func Investigate(paths []string, opts Options) (Report, error) { + if opts.GroupBy == "" { + opts.GroupBy = "user" + } + if opts.GroupBy != "user" && opts.GroupBy != "session" || opts.Top < 1 || opts.Top > 100 { + return Report{}, errors.New("Claude logs support group-by user or session and top 1-100") + } + if len(opts.SourceScope) > 256 || opts.SecretEnv != "" && strings.TrimSpace(opts.SourceScope) == "" { + return Report{}, errors.New("a repeatable hashing secret requires an explicit source-scope (at most 256 bytes)") + } + data, err := load(paths) + if err != nil { + return Report{}, err + } + key, mode, err := attribution.Secret(opts.SecretEnv) + if err != nil { + return Report{}, err + } + defer clear(key) + p, selection, err := periods(opts, data, key) + if err != nil { + return Report{}, err + } + c := &catalog{key: key, scope: opts.SourceScope, entries: map[uint64]label{}} + w := [2]*aggregate{newAggregate(p[0], true), newAggregate(p[1], true)} + r := Report{Schema: "fleetdiff-claude-code-records/v1", Unit: "observed api_request records", Finality: "unknown", AccountingID: accounting.ID, Hashing: mode, GroupBy: opts.GroupBy, Deduplication: "retain exact copies", Models: []Model{}, ComparisonChecks: []string{}} + r.PeriodSelection = selection + r.InputFiles = len(paths) + if opts.DeduplicateExact { + r.Deduplication = "remove exact exported copies only" + } + seen := map[[32]byte]struct{}{} + prompts := map[[32]byte][2]uint64{} + models := map[[32]byte][2]*aggregate{} + var modelOrder [][32]byte + profiles := [2]map[[32]byte]string{{}, {}} + r.ReadRecords, r.IgnoredEvents, err = read(data, key, func(row record) error { + side := -1 + for i, period := range p { + if !row.time.Before(period.Start) && row.time.Before(period.End) { + side = i + break + } + } + if side < 0 { + r.OutsidePeriods++ + return nil + } + a := w[side] + if _, ok := seen[row.fingerprint]; ok { + a.w.DuplicateCopies++ + if opts.DeduplicateExact { + return nil + } + } + seen[row.fingerprint] = struct{}{} + switch row.event { + case "claude_code.api_error": + a.w.ErrorEvents++ + a.w.ErrorStatus[errorStatus(row)]++ + return nil + case "claude_code.api_retries_exhausted": + a.w.ExhaustedEvents++ + return nil + } + o := normalize(row) + if err := a.add(o); err != nil { + return err + } + unverified := false + for j, name := range []string{"service.version", "fleetdiff.capture.profile"} { + alias := "unavailable" + if v, ok := row.text(name); ok { + d := attribution.Digest(key, name, []byte(v)) + alias = profiles[j][d] + if alias == "" { + if len(profiles[j]) >= MaxProfiles { + return errors.New("capture exceeds 128 version/profile values") + } + alias = fmt.Sprintf("source-%d", len(profiles[j])+1) + profiles[j][d] = alias + } + if j == 0 && v != "2.1.296" || j == 1 && v != "claude-code-logs/v1" { + unverified = true + } + } else { + unverified = true + } + if j == 0 { + a.w.Versions[alias]++ + } else { + a.w.Profiles[alias]++ + } + } + if unverified { + a.w.Quality["source_profile_unverified"]++ + } + h, kind, ok, err := c.group(row, opts.GroupBy) + if err != nil { + return err + } + a.w.IdentityKinds[kind]++ + coverage := a.w.IdentityCoverage[kind] + coverage.Records++ + if o.total != nil { + coverage.TokenRecords++ + } + if o.cost != nil { + coverage.CostRecords++ + } + a.w.IdentityCoverage[kind] = coverage + if _, present := row.resolve("identity.source"); present { + a.w.Quality["unverified_identity_regime"]++ + } + if !ok { + a.w.Unattributed++ + } else { + if o.total != nil { + if err := a.top.AddHash(h, int64(*o.total)); err != nil { + return errors.New("cannot add token ranking weight") + } + } + if o.input != nil { + if err := a.cacheTop.AddHash(h, int64(*o.input-*o.read)); err != nil { + return errors.New("cannot add uncached ranking weight") + } + } + } + if d, ok, err := c.prompt(row); err != nil { + return err + } else if ok { + counts, exists := prompts[d] + if !exists && len(prompts) >= MaxIdentities { + return errors.New("capture exceeds 250000 scoped prompts; select a smaller capture") + } + if counts[side] == 0 { + a.w.Prompts++ + } + if counts[side] == 1 { + a.w.MultipleRecordPrompts++ + } + counts[side]++ + prompts[d] = counts + a.w.PromptRecords++ + } + model, ok := row.text("model") + d := attribution.Digest(key, "model", []byte(model)) + if !ok { + a.w.Quality["model_missing"]++ + return nil + } + m, exists := models[d] + if !exists { + if len(models) >= MaxProfiles { + return errors.New("capture exceeds 128 model values") + } + m = [2]*aggregate{newAggregate(p[0], false), newAggregate(p[1], false)} + models[d] = m + modelOrder = append(modelOrder, d) + } + return m[side].add(o) + }) + if err != nil { + return Report{}, err + } + r.Before, r.After = w[0].result(), w[1].result() + if r.Before.Records+r.After.Records+r.Before.ErrorEvents+r.After.ErrorEvents+r.Before.ExhaustedEvents+r.After.ExhaustedEvents == 0 { + return Report{}, errors.New("no recognized Claude API events in the selected periods") + } + for i, d := range modelOrder { + m := models[d] + r.Models = append(r.Models, Model{fmt.Sprintf("model-%d", i+1), m[0].result(), m[1].result()}) + } + r.Rankings, err = c.rank("observed_tokens", w[0], w[1], false, opts) + if err != nil { + return Report{}, err + } + r.UncachedRankings, err = c.rank("uncached_input", w[0], w[1], true, opts) + if err != nil { + return Report{}, err + } + r.ComparisonChecks = checks(r.Before, r.After, opts.DeduplicateExact) + if len(r.ComparisonChecks) == 0 && r.Before.Total.Value != nil && r.After.Total.Value != nil { + v := compare.Decompose(*r.Before.Total.Value, *r.After.Total.Value, r.Before.Records, r.After.Records) + if v != nil { + r.Decomposition = &Decomposition{v.RequestContribution, v.TokensPerRequestContribution} + } + } + return r, nil +} + +func sameMix(a, b map[string]uint64, n, m uint64) bool { + for k, v := range a { + if v*m != b[k]*n { + return false + } + } + for k, v := range b { + if v*n != a[k]*m { + return false + } + } + return true +} +func checks(a, b Window, dedup bool) []string { + var out = []string{} + if a.Records == 0 || b.Records == 0 { + out = append(out, "period_without_api_request_records") + } + if a.Total.Records != a.Records || b.Total.Records != b.Records { + out = append(out, "incomplete_token_components") + } + if a.Quality["source_profile_unverified"]+b.Quality["source_profile_unverified"] > 0 { + out = append(out, "unverified_source_profile") + } + if !sameMix(a.Versions, b.Versions, a.Records, b.Records) { + out = append(out, "source_version_mix_changed") + } + if !sameMix(a.Profiles, b.Profiles, a.Records, b.Records) { + out = append(out, "capture_profile_mix_changed") + } + if !sameMix(a.IdentityKinds, b.IdentityKinds, a.Records, b.Records) { + out = append(out, "identity_source_mix_changed") + } + if a.PromptRecords != a.Records || b.PromptRecords != b.Records { + out = append(out, "incomplete_prompt_identity") + } + if a.PromptRecords*b.Prompts != b.PromptRecords*a.Prompts || a.MultipleRecordPrompts*b.Prompts != b.MultipleRecordPrompts*a.Prompts { + out = append(out, "records_per_prompt_changed") + } + if a.ErrorEvents*b.Records != b.ErrorEvents*a.Records || a.ExhaustedEvents*b.Records != b.ExhaustedEvents*a.Records { + out = append(out, "error_or_exhaustion_mix_changed") + } + if !sameMix(a.ErrorStatus, b.ErrorStatus, a.ErrorEvents, b.ErrorEvents) { + out = append(out, "error_status_mix_changed") + } + if a.Unattributed*b.Records != b.Unattributed*a.Records { + out = append(out, "attribution_coverage_changed") + } + if !dedup && a.DuplicateCopies+b.DuplicateCopies > 0 { + out = append(out, "exact_export_copies_retained") + } + return out +} + +func errorStatus(r record) string { + v, ok := r.resolve("status_code") + if !ok { + return "missing" + } + n, ok := number(v) + if !ok || n < 100 || n > 599 { + return "invalid" + } + if n == 429 { + return "http_429" + } + if n >= 500 { + return "http_5xx" + } + if n >= 400 { + return "other_http_4xx" + } + return "other_http_status" +} diff --git a/internal/clauderecords/investigate_test.go b/internal/clauderecords/investigate_test.go new file mode 100644 index 0000000..d3db2eb --- /dev/null +++ b/internal/clauderecords/investigate_test.go @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/llm-measurement/fleetdiff/internal/accounting" + "github.com/llm-measurement/fleetdiff/internal/boundedjson" +) + +func testRow(day int) record { + r := record{event: "claude_code.api_request", time: time.Date(2026, 10, day, 12, 0, 0, 0, time.UTC), attrs: map[string]value{}, resource: map[string]value{}} + for k, v := range map[string]string{"input_tokens": "7", "cache_read_tokens": "11", "cache_creation_tokens": "5", "output_tokens": "4", "cost_usd": "0.00003435", "cost_usd_micros": "34", "model": "private-model", "user.id": "private-installation", "session.id": "private-session", "prompt.id": "private-prompt", "service.version": "2.1.296", "fleetdiff.capture.profile": "claude-code-logs/v1"} { + r.attrs[k] = value{kind: "stringValue", text: v} + } + return r +} +func testEnvelope(rows ...record) []byte { + if len(rows) > 500 { + return append(append(testEnvelope(rows[:500]...), '\n'), testEnvelope(rows[500:]...)...) + } + attrs := func(m map[string]value) []any { + out := []any{} + var keys []string + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + for _, k := range keys { + v := m[k] + out = append(out, map[string]any{"key": k, "value": map[string]any{v.kind: v.text}}) + } + return out + } + var resources []any + for _, r := range rows { + resources = append(resources, map[string]any{"resource": map[string]any{"attributes": attrs(r.resource)}, "scopeLogs": []any{map[string]any{"logRecords": []any{map[string]any{"timeUnixNano": strconv.FormatInt(r.time.UnixNano(), 10), "body": map[string]any{"stringValue": r.event}, "attributes": attrs(r.attrs)}}}}}) + } + b, _ := json.Marshal(map[string]any{"resourceLogs": resources}) + return b +} +func testFile(t *testing.T, b []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "capture.jsonl") + if err := os.WriteFile(path, b, 0600); err != nil { + t.Fatal(err) + } + return path +} +func testOptions() Options { + return Options{BeforePeriod: "2026-10-11", AfterPeriod: "2026-10-12", Top: 20, Now: time.Date(2026, 10, 14, 0, 0, 0, 0, time.UTC)} +} +func investigateFile(path string, opts Options) (Report, error) { + return Investigate([]string{path}, opts) +} +func report(t *testing.T, rows ...record) Report { + t.Helper() + r, e := investigateFile(testFile(t, testEnvelope(rows...)), testOptions()) + if e != nil { + t.Fatal(e) + } + return r +} +func TestNativeTokenAndCostNormalization(t *testing.T) { + r := report(t, testRow(11), testRow(12)) + w := r.Before + if *w.Input.Value != 23 || *w.Output.Value != 4 || *w.Total.Value != 27 || *w.Cache.Read.Value != 11 || *w.Cache.Write.Value != 5 || *w.Cache.Uncached.Value != 12 { + t.Fatalf("wrong inclusive mapping: %+v", w) + } + if *w.Cost != "0.00003435" || w.Quality["cost_micros_fallback"] != 0 { + t.Fatalf("rounded native cost: %+v", w) + } + if w.DecimalMinusMicros == nil || *w.DecimalMinusMicros != "0.00000035" || w.PairedCosts != 1 { + t.Fatal(w) + } + if r.Finality != "unknown" || r.Unit != "observed api_request records" || r.Decomposition == nil || len(r.ComparisonChecks) != 0 { + t.Fatalf("wrong unit/checks: %+v", r) + } + if len(r.Rankings.Movers) != 1 || r.Rankings.Movers[0].Item != "installation-1" { + t.Fatal(r.Rankings) + } +} +func TestIncompleteInvalidAndZeroUsage(t *testing.T) { + for _, tc := range []struct { + name, field, replacement, quality string + remove bool + }{ + {"missing cache", "cache_read_tokens", "", "cache_read_tokens_missing", true}, + {"negative", "input_tokens", "-1", "input_tokens_invalid", false}, + {"fractional", "output_tokens", "1.5", "output_tokens_invalid", false}, + {"overflow", "output_tokens", "9223372036854775808", "output_tokens_invalid", false}, + {"row sum", "input_tokens", "9223372036854775807", "input_sum_invalid", false}, + } { + t.Run(tc.name, func(t *testing.T) { + a, b := testRow(11), testRow(12) + if tc.remove { + delete(b.attrs, tc.field) + } else { + b.attrs[tc.field] = value{kind: "stringValue", text: tc.replacement} + } + r := report(t, a, b) + if r.After.Total.Value != nil || r.After.Quality[tc.quality] != 1 || !slices.Contains(r.ComparisonChecks, "incomplete_token_components") || r.Decomposition != nil { + t.Fatalf("invalid row treated as complete: %+v", r.After) + } + }) + } + a, b := testRow(11), testRow(12) + for _, k := range []string{"input_tokens", "cache_read_tokens", "cache_creation_tokens", "output_tokens"} { + b.attrs[k] = value{kind: "intValue", text: "0"} + } + r := report(t, a, b) + if r.After.Total.Value == nil || *r.After.Total.Value != 0 || r.After.Quality["zero_token_origin_unknown"] != 1 { + t.Fatal(r.After) + } +} +func TestDecimalAndMicrosCosts(t *testing.T) { + for _, tc := range []struct { + name, decimal, micros string + remove bool + want string + quality string + }{ + {"prefer decimal", "0.00003435", "34", false, "0.00003435", ""}, + {"fallback", "", "34", true, "0.000034", "cost_micros_fallback"}, + {"bad decimal no fallback", "SECRET", "34", false, "", "cost_invalid"}, + {"bad micros", "", "-1", true, "", "cost_invalid"}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testRow(12) + b.attrs["cost_usd"] = value{kind: "stringValue", text: tc.decimal} + b.attrs["cost_usd_micros"] = value{kind: "intValue", text: tc.micros} + if tc.remove { + delete(b.attrs, "cost_usd") + } + r := report(t, testRow(11), b) + if tc.want == "" { + if r.After.Cost != nil { + t.Fatal(*r.After.Cost) + } + } else if r.After.Cost == nil || *r.After.Cost != tc.want { + t.Fatal(r.After.Cost) + } + if tc.quality != "" && r.After.Quality[tc.quality] != 1 { + t.Fatal(r.After.Quality) + } + }) + } +} +func TestAggregateOverflowFailsNotRowOverflow(t *testing.T) { + a := testRow(11) + for _, k := range []string{"cache_read_tokens", "cache_creation_tokens", "output_tokens"} { + a.attrs[k] = value{kind: "intValue", text: "0"} + } + a.attrs["input_tokens"] = value{kind: "intValue", text: strconv.FormatInt(math.MaxInt64, 10)} + _, err := investigateFile(testFile(t, testEnvelope(a, a, testRow(12))), testOptions()) + if err == nil || !strings.Contains(err.Error(), "supported range") { + t.Fatal(err) + } +} +func TestDuplicateExportsVersusRepeatedPrompts(t *testing.T) { + a, b, c := testRow(11), testRow(12), testRow(12) + c.time = c.time.Add(time.Second) + path := testFile(t, testEnvelope(a, b, b, c)) + o := testOptions() + r, err := investigateFile(path, o) + if err != nil { + t.Fatal(err) + } + if r.After.Records != 3 || r.After.DuplicateCopies != 1 || r.After.Prompts != 1 || !slices.Contains(r.ComparisonChecks, "exact_export_copies_retained") { + t.Fatal(r.After) + } + o.DeduplicateExact = true + r, err = investigateFile(path, o) + if err != nil { + t.Fatal(err) + } + if r.After.Records != 2 || r.After.DuplicateCopies != 1 || r.After.MultipleRecordPrompts != 1 || !slices.Contains(r.ComparisonChecks, "records_per_prompt_changed") { + t.Fatal(r.After) + } +} +func TestRetryCompanionsAndSilentPeriod(t *testing.T) { + a, e, x := testRow(11), testRow(12), testRow(12) + e.event = "claude_code.api_error" + x.event = "claude_code.api_retries_exhausted" + r := report(t, a, e, x) + if r.After.Records != 0 || r.After.Total.Value != nil || r.After.Cost != nil || r.After.ErrorEvents != 1 || r.After.ExhaustedEvents != 1 || r.Decomposition != nil { + t.Fatalf("companions counted as usage: %+v", r.After) + } + if !slices.Contains(r.ComparisonChecks, "period_without_api_request_records") { + t.Fatal(r.ComparisonChecks) + } +} +func TestErrorCategoriesDoNotCollapse429And500(t *testing.T) { + a, b, x, y := testRow(11), testRow(12), testRow(11), testRow(12) + x.event = "claude_code.api_error" + y.event = x.event + x.attrs["status_code"] = value{kind: "intValue", text: "429"} + y.attrs["status_code"] = value{kind: "intValue", text: "500"} + r := report(t, a, b, x, y) + if !slices.Contains(r.ComparisonChecks, "error_status_mix_changed") || r.Before.ErrorStatus["http_429"] != 1 || r.After.ErrorStatus["http_5xx"] != 1 || r.Decomposition != nil { + t.Fatal(r) + } +} + +func TestIdentityPriorityLocalityAndScope(t *testing.T) { + a, b := testRow(11), testRow(12) + a.attrs["user.account_uuid"] = value{kind: "stringValue", text: "private-account"} + b.resource["user.account_uuid"] = a.attrs["user.account_uuid"] + r := report(t, a, b) + if r.Rankings.Movers[0].Item != "account-1" || len(r.Rankings.Movers) != 1 { + t.Fatal(r.Rankings) + } + b.attrs["user.account_uuid"] = value{kind: "stringValue", text: "other-private-account"} + r = report(t, a, b) + if r.After.Unattributed != 1 || r.After.PromptRecords != 0 { + t.Fatal(r.After) + } + b = testRow(12) + delete(b.attrs, "user.id") + o := testOptions() + o.GroupBy = "session" + r, e := investigateFile(testFile(t, testEnvelope(a, b)), o) + if e != nil { + t.Fatal(e) + } + if r.After.Unattributed != 1 { + t.Fatal(r.After) + } + t.Setenv("CLAUDE_TEST_KEY", "fixed-test-secret-at-least-32-bytes") + o.SecretEnv = "CLAUDE_TEST_KEY" + if _, e = investigateFile(testFile(t, testEnvelope(a)), o); e == nil { + t.Fatal("unscoped persistent key accepted") + } + o.SourceScope = "app-a" + o.ShowHashes = true + a.attrs["user.id"] = value{kind: "stringValue", text: "x"} + p := testFile(t, testEnvelope(a, testRow(12))) + r, e = investigateFile(p, o) + if e != nil { + t.Fatal(e) + } + o.SourceScope = "app-b" + s, e := investigateFile(p, o) + if e != nil { + t.Fatal(e) + } + if r.Rankings.Movers[0].Hash == s.Rankings.Movers[0].Hash { + t.Fatal("scope did not separate identities") + } +} + +func TestUnverifiedOIDCAttributionIsNotInstallation(t *testing.T) { + a, b := testRow(11), testRow(12) + b.attrs["identity.source"] = value{kind: "stringValue", text: "gateway_oidc"} + r := report(t, a, b) + if r.After.Unattributed != 1 || r.After.PromptRecords != 0 || r.After.IdentityKinds["installation"] != 0 || r.After.Quality["unverified_identity_regime"] != 1 { + t.Fatal(r.After) + } + b = testRow(12) + b.attrs["user.account_uuid"] = value{kind: "stringValue", text: " "} + b.attrs["user.email"] = value{kind: "stringValue", text: "private@example.invalid"} + r = report(t, a, b) + if r.After.IdentityKinds["email"] != 1 { + t.Fatal(r.After) + } +} +func TestComparisonMixChecks(t *testing.T) { + a, b := testRow(11), testRow(12) + b.attrs["service.version"] = value{kind: "stringValue", text: "PRIVATE_VERSION"} + b.attrs["fleetdiff.capture.profile"] = value{kind: "stringValue", text: "OTHER_PROFILE"} + b.attrs["user.email"] = value{kind: "stringValue", text: "private@example.invalid"} + r := report(t, a, b) + for _, want := range []string{"unverified_source_profile", "source_version_mix_changed", "capture_profile_mix_changed", "identity_source_mix_changed"} { + if !slices.Contains(r.ComparisonChecks, want) { + t.Fatalf("missing %s: %v", want, r.ComparisonChecks) + } + } + j, _ := json.Marshal(r) + for _, s := range []string{"PRIVATE_VERSION", "OTHER_PROFILE", "private@example.invalid", "private-model", "private-installation", "private-session", "private-prompt"} { + if strings.Contains(string(j), s) { + t.Fatalf("leaked %s", s) + } + } +} +func TestAliasOrderStableAcrossRandomKeys(t *testing.T) { + a, b, c := testRow(11), testRow(12), testRow(12) + b.attrs["user.id"] = value{kind: "stringValue", text: "second"} + c.attrs["user.id"] = value{kind: "stringValue", text: "third"} + p := testFile(t, testEnvelope(a, b, c)) + var want []string + for i := 0; i < 10; i++ { + r, e := investigateFile(p, testOptions()) + if e != nil { + t.Fatal(e) + } + var got []string + for _, m := range r.Rankings.Movers { + got = append(got, m.Item) + } + if i == 0 { + want = got + } else if !reflect.DeepEqual(want, got) { + t.Fatalf("unstable aliases: %v vs %v", want, got) + } + } +} +func TestRankingBoundsCoverExactTotals(t *testing.T) { + var rows []record + for i := 0; i < 650; i++ { + r := testRow(11) + r.attrs["user.id"] = value{kind: "stringValue", text: fmt.Sprint(i)} + rows = append(rows, r) + r = testRow(12) + r.attrs["user.id"] = value{kind: "stringValue", text: fmt.Sprint(i)} + r.attrs["output_tokens"] = value{kind: "intValue", text: "14"} + rows = append(rows, r) + } + r := report(t, rows...) + for _, m := range r.Rankings.Movers { + if m.Before.Lower > 27 || m.Before.Upper < 27 || m.After.Lower > 37 || m.After.Upper < 37 || m.Delta.Lower > 10 || m.Delta.Upper < 10 { + t.Fatal(m) + } + } + if r.Rankings.AfterMaxError == 0 { + t.Fatal("fixture did not exercise approximate ranking") + } +} +func TestReaderRejectsUnsafeOrAmbiguousInput(t *testing.T) { + for _, b := range [][]byte{[]byte(`{"resourceLogs":[],"resourceLogs":[]}`), []byte(`{"resourceLogs":[`), []byte(strings.Repeat("[", 40) + strings.Repeat("]", 40)), []byte(`{"error":"PRIVATE"}`), append(testEnvelope(testRow(11)), []byte("PRIVATE")...)} { + _, e := investigateFile(testFile(t, b), testOptions()) + if e == nil || strings.Contains(e.Error(), "PRIVATE") { + t.Fatal(e) + } + } + r := testRow(11) + r.time = time.Time{} + if _, e := investigateFile(testFile(t, testEnvelope(r)), testOptions()); e == nil { + t.Fatal("invalid timestamp accepted") + } + path := filepath.Join(t.TempDir(), "private-link") + if e := os.Symlink(testFile(t, testEnvelope(testRow(11))), path); e != nil { + t.Fatal(e) + } + if _, e := investigateFile(path, testOptions()); e == nil { + t.Fatal("symlink accepted") + } +} +func TestNumericRepresentationsUseExactIntegers(t *testing.T) { + for _, kind := range []string{"stringValue", "intValue", "doubleValue"} { + v := value{kind: kind, text: "9007199254740993"} + n, ok := number(v) + if !ok || n != 9007199254740993 { + t.Fatalf("rounded %s: %d", kind, n) + } + } + for _, text := range []string{"9e18", "9007199254740993", "4.0"} { + if _, ok := number(value{kind: "doubleValue", text: text}); !ok { + t.Fatal(text) + } + } + for _, text := range []string{"NaN", "+Inf", "1e-999999999", "1e999999999", "0x1p2", "1/2", "9223372036854775808", "0.1", "-1"} { + if _, ok := number(value{kind: "doubleValue", text: text}); ok { + t.Fatalf("accepted %s", text) + } + } + r := testRow(11) + r.attrs["output_tokens"] = value{kind: "doubleValue", text: "4.0"} + o := normalize(r) + if o.total == nil || *o.total != 27 { + t.Fatal(o) + } + if accounting.CostText(o.cost) != "0.00003435" { + t.Fatal(o.cost) + } +} +func FuzzClaudeEnvelope(f *testing.F) { + f.Add(testEnvelope(testRow(11))) + f.Add([]byte(`{"resourceLogs":[]}`)) + f.Add([]byte(`{"resourceLogs":[null]}`)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 1<<20 { + t.Skip() + } + v, e := boundedjson.Decode(b, 32, 250000) + if e != nil { + return + } + _, _, _ = envelope(v, []byte("test-key"), MaxRecords, func(r record) error { _ = normalize(r); return nil }) + }) +} diff --git a/internal/clauderecords/period.go b/internal/clauderecords/period.go new file mode 100644 index 0000000..dc4b42d --- /dev/null +++ b/internal/clauderecords/period.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "errors" + "math" + "time" + + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +func periods(o Options, data [][]byte, key []byte) ([2]spend.Period, string, error) { + var p [2]spend.Period + selection := "explicit" + if o.Now.IsZero() { + o.Now = time.Now() + } + if o.BeforePeriod != "" || o.AfterPeriod != "" { + for i, s := range []string{o.BeforePeriod, o.AfterPeriod} { + v, err := spend.ParsePeriod(s) + if err != nil { + return p, selection, errors.New("supply both before-period and after-period as UTC dates or RFC3339 start/end") + } + p[i] = v + } + } else { + var earliest, latest time.Time + _, _, err := read(data, key, func(r record) error { + if earliest.IsZero() || r.time.Before(earliest) { + earliest = r.time + } + if latest.IsZero() || r.time.After(latest) { + latest = r.time + } + return nil + }) + if err != nil { + return p, "", err + } + if earliest.IsZero() { + return p, "", errors.New("no recognized Claude API events; use the collector JSON logs recipe") + } + start := earliest.Truncate(24 * time.Hour) + if !start.Equal(earliest) { + start = start.AddDate(0, 0, 1) + } + end := latest.Truncate(24 * time.Hour) + if today := o.Now.UTC().Truncate(24 * time.Hour); end.After(today) { + end = today + } + if end.Sub(start) >= 14*24*time.Hour { + p = [2]spend.Period{{Start: end.AddDate(0, 0, -14), End: end.AddDate(0, 0, -7)}, {Start: end.AddDate(0, 0, -7), End: end}} + selection = "complete_utc_weeks" + } else { + if !earliest.Before(latest) || latest.UnixNano() > math.MaxInt64-2 { + return p, "", errors.New("observed span cannot form two periods; supply explicit before-period and after-period") + } + // Include the last nanosecond and round outwards by at most one more + // nanosecond so the two observed-span periods have equal duration. + end = latest.Add(time.Nanosecond) + if end.Sub(earliest)%2 != 0 { + end = end.Add(time.Nanosecond) + } + middle := earliest.Add(end.Sub(earliest) / 2) + p = [2]spend.Period{{Start: earliest, End: middle}, {Start: middle, End: end}} + selection = "observed_span_halves" + } + } + if p[0].End.After(p[1].Start) || p[0].End.Sub(p[0].Start) != p[1].End.Sub(p[1].Start) || p[1].End.After(o.Now) { + return p, selection, errors.New("select equal completed periods, with before ending at or before after starts") + } + return p, selection, nil +} diff --git a/internal/clauderecords/period_test.go b/internal/clauderecords/period_test.go new file mode 100644 index 0000000..9ce98f1 --- /dev/null +++ b/internal/clauderecords/period_test.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +func TestDefaultPeriodsAndObservedSpan(t *testing.T) { + opts := testOptions() + opts.BeforePeriod, opts.AfterPeriod = "", "" + rows := []record{testRow(11), testRow(12)} + r, err := investigateFile(testFile(t, testEnvelope(rows...)), opts) + if err != nil { + t.Fatal(err) + } + if r.PeriodSelection != "observed_span_halves" || r.Before.Records != 1 || r.After.Records != 1 || !r.Before.Period.Start.Equal(rows[0].time) || !r.After.Period.End.After(rows[1].time) { + t.Fatal(r) + } + if r.Before.Period.End.Sub(r.Before.Period.Start) != r.After.Period.End.Sub(r.After.Period.Start) { + t.Fatal("unequal observed halves") + } + // A record at the exact midpoint belongs to after, with no overlap. + middle := testRow(11) + middle.time = r.Before.Period.End + r, err = investigateFile(testFile(t, testEnvelope(rows[0], middle, rows[1])), opts) + if err != nil || r.Before.Records != 1 || r.After.Records != 2 { + t.Fatal(r, err) + } + // Complete weeks use the same conservative interior-day rule as spend logs. + rows = []record{testRow(1), testRow(2), testRow(8), testRow(9), testRow(15), testRow(16)} + opts.Now = time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC) + r, err = investigateFile(testFile(t, testEnvelope(rows...)), opts) + if err != nil { + t.Fatal(err) + } + if r.PeriodSelection != "complete_utc_weeks" || r.Before.Period.Start.Day() != 2 || r.After.Period.Start.Day() != 9 || r.After.Period.End.Day() != 16 || r.Before.Records != 2 || r.After.Records != 2 || r.OutsidePeriods != 2 { + t.Fatal(r) + } +} + +func TestDefaultPeriodsNeedUsableTimestampsAndExplicitOverride(t *testing.T) { + opts := testOptions() + opts.BeforePeriod, opts.AfterPeriod = "", "" + if _, err := investigateFile(testFile(t, testEnvelope(testRow(11))), opts); err == nil || !strings.Contains(err.Error(), "explicit") { + t.Fatal(err) + } + if _, err := investigateFile(testFile(t, testEnvelope(testRow(14), testRow(15))), opts); err == nil { + t.Fatal("future fallback accepted") + } + opts.BeforePeriod = "2026-10-11" + if _, err := investigateFile(testFile(t, testEnvelope(testRow(11), testRow(12))), opts); err == nil { + t.Fatal("one explicit period accepted") + } + opts.AfterPeriod = "2026-10-12" + r, err := investigateFile(testFile(t, testEnvelope(testRow(11), testRow(12), testRow(1))), opts) + if err != nil || r.PeriodSelection != "explicit" || r.OutsidePeriods != 1 { + t.Fatal(r, err) + } +} + +func TestRotatedFilesMatchOneFileAndDeduplicateAcrossFiles(t *testing.T) { + t.Setenv("CLAUDE_ROTATION_KEY", "synthetic-rotation-regression-key-32bytes") + opts := testOptions() + opts.SecretEnv, opts.SourceScope, opts.ShowHashes = "CLAUDE_ROTATION_KEY", "synthetic-source", true + a, b := testRow(11), testRow(12) + paths := []string{testFile(t, testEnvelope(a)), testFile(t, testEnvelope(b)), testFile(t, nil)} + got, err := Investigate(paths, opts) + if err != nil { + t.Fatal(err) + } + want, err := investigateFile(testFile(t, append(append(testEnvelope(a), '\n'), testEnvelope(b)...)), opts) + if err != nil { + t.Fatal(err) + } + if got.InputFiles != 3 { + t.Fatal(got.InputFiles) + } + got.InputFiles = 1 + if !reflect.DeepEqual(got, want) { + t.Fatal("rotation changed report") + } + paths = append(paths, paths[1]) + got, err = Investigate(paths, opts) + if err != nil || got.After.Records != 2 || got.After.DuplicateCopies != 1 { + t.Fatal(got.After, err) + } + opts.DeduplicateExact = true + got, err = Investigate(paths, opts) + if err != nil || got.After.Records != 1 || got.After.DuplicateCopies != 1 { + t.Fatal(got.After, err) + } + opts.BeforePeriod, opts.AfterPeriod = "", "" + got, err = Investigate(paths, opts) + if err != nil || got.PeriodSelection != "observed_span_halves" || got.Before.Records != 1 || got.After.Records != 1 { + t.Fatal(got, err) + } +} + +func TestRotatedInputLimitsAndSnapshot(t *testing.T) { + if _, err := load(nil); err == nil { + t.Fatal("empty file set") + } + if _, err := load(make([]string, MaxFiles+1)); err == nil { + t.Fatal("file count unbounded") + } + dir := t.TempDir() + small := testFile(t, testEnvelope(testRow(11))) + large := filepath.Join(dir, "private-path-sentinel.jsonl") + f, err := os.Create(large) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(MaxFileBytes); err != nil { + t.Fatal(err) + } + f.Close() + if _, err := load([]string{small, large}); err == nil || strings.Contains(err.Error(), "sentinel") { + t.Fatal("combined byte cap/privacy", err) + } + if err := os.Symlink(small, filepath.Join(dir, "symlink")); err != nil { + t.Fatal(err) + } + if _, err := load([]string{small, filepath.Join(dir, "symlink")}); err == nil { + t.Fatal("second-file symlink accepted") + } + data, err := load([]string{small}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(small, testEnvelope(testRow(12)), 0600); err != nil { + t.Fatal(err) + } + var day int + _, _, err = read(data, nil, func(r record) error { day = r.time.Day(); return nil }) + if err != nil || day != 11 { + t.Fatal("snapshot changed after rotation", day, err) + } + _, _, err = readFile(append(append(testEnvelope(testRow(11)), '\n'), testEnvelope(testRow(12))...), nil, 1, func(record) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "log records") { + t.Fatal("record cap resets at envelope", err) + } + // Across files, ignored events count toward the same record cap too. + ignored := []byte(`{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":{"stringValue":"ignored"}}]}]}]}`) + inputs := make([][]byte, MaxRecords+1) + for i := range inputs { + inputs[i] = ignored + } + if _, _, err := read(inputs, nil, func(record) error { return nil }); err == nil { + t.Fatal("record cap resets across files") + } +} diff --git a/internal/clauderecords/reader.go b/internal/clauderecords/reader.go new file mode 100644 index 0000000..c39aef9 --- /dev/null +++ b/internal/clauderecords/reader.go @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +// Package clauderecords compares observed Claude Code api_request log records. +package clauderecords + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/llm-measurement/fleetdiff/internal/attribution" + "github.com/llm-measurement/fleetdiff/internal/boundedjson" + "github.com/llm-measurement/fleetdiff/internal/localfile" +) + +const ( + MaxFileBytes = 128 << 20 + MaxEnvelopeBytes = 8 << 20 + MaxRecords = 1_000_000 + MaxIdentities = 250_000 + MaxProfiles = 128 + MaxFiles = 256 +) + +type value struct { + kind, text string + invalid bool +} +type record struct { + event string + time time.Time + attrs, resource map[string]value + fingerprint [32]byte +} + +var allowed = map[string]bool{ + "input_tokens": true, "output_tokens": true, "cache_read_tokens": true, "cache_creation_tokens": true, + "cost_usd": true, "cost_usd_micros": true, "model": true, "status_code": true, "attempt": true, + "user.account_uuid": true, "user.email": true, "user.id": true, "session.id": true, "prompt.id": true, + "identity.source": true, "service.version": true, "fleetdiff.capture.profile": true, +} + +func object(v any) (map[string]any, error) { + m, ok := v.(map[string]any) + if !ok { + return nil, errors.New("expected an OTLP JSON object") + } + return m, nil +} + +func children(m map[string]any, name string, limit int) ([]any, error) { + v, present := m[name] + if !present { + return nil, nil + } + a, ok := v.([]any) + if !ok || len(a) > limit { + return nil, errors.New("invalid or over-limit OTLP repeated field") + } + return a, nil +} + +func attributes(m map[string]any) (map[string]value, error) { + a, err := children(m, "attributes", 256) + if err != nil { + return nil, err + } + out, seen := map[string]value{}, map[string]bool{} + for _, item := range a { + kv, err := object(item) + if err != nil { + return nil, err + } + key, ok := kv["key"].(string) + if !ok || key == "" || len(key) > 256 || seen[key] { + return nil, errors.New("invalid or duplicate OTLP attribute") + } + seen[key] = true + if !allowed[key] { + continue + } + v, ok := kv["value"].(map[string]any) + if !ok || len(v) != 1 { + out[key] = value{invalid: true} + continue + } + for kind, raw := range v { + item := value{kind: kind} + switch x := raw.(type) { + case string: + item.text = x + case json.Number: + item.text = x.String() + if kind == "stringValue" { + item.invalid = true + } + default: + item.invalid = true + } + if kind != "stringValue" && kind != "intValue" && kind != "doubleValue" || len(item.text) > 4096 { + item.invalid = true + item.text = "" + } + out[key] = item + } + } + return out, nil +} + +// Keep a bounded snapshot: rotation cannot change bytes between period selection +// and accounting. Limits apply to the entire invocation, not to each file. +func load(paths []string) ([][]byte, error) { + if len(paths) == 0 || len(paths) > MaxFiles { + return nil, errors.New("select 1-256 collector JSONL files") + } + data := make([][]byte, 0, len(paths)) + remaining := MaxFileBytes + for i, path := range paths { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return nil, fmt.Errorf("capture file %d: cannot open capture directory", i+1) + } + b, err := localfile.Read(root, filepath.Base(path), remaining) + root.Close() + if err != nil { + return nil, fmt.Errorf("capture file %d: %w", i+1, err) + } + data = append(data, b) + remaining -= len(b) + } + return data, nil +} + +func read(data [][]byte, key []byte, visit func(record) error) (uint64, uint64, error) { + var total, ignored uint64 + for i, b := range data { + n, skip, err := readFile(b, key, MaxRecords-total, visit) + total += n + ignored += skip + if err != nil { + return total, ignored, fmt.Errorf("capture file %d: %w", i+1, err) + } + } + if total == 0 { + return 0, 0, errors.New("capture contains no log records; use the collector JSON logs recipe") + } + return total, ignored, nil +} + +func readFile(data, key []byte, remaining uint64, visit func(record) error) (uint64, uint64, error) { + scan := bufio.NewScanner(bytes.NewReader(data)) + scan.Buffer(make([]byte, 64<<10), MaxEnvelopeBytes+1) + var total, ignored uint64 + line := 0 + for scan.Scan() { + line++ + b := bytes.TrimSpace(scan.Bytes()) + if len(b) == 0 { + continue + } + if len(b) > MaxEnvelopeBytes { + return total, ignored, errors.New("OTLP envelope exceeds 8 MiB") + } + v, err := boundedjson.Decode(b, 32, 250000) + if err != nil { + return total, ignored, fmt.Errorf("capture envelope %d: invalid or over-limit JSON", line) + } + n, skip, err := envelope(v, key, remaining-total, visit) + total += n + ignored += skip + if err != nil { + return total, ignored, fmt.Errorf("capture envelope %d: %s", line, err) + } + } + if err := scan.Err(); err != nil { + return total, ignored, errors.New("cannot read capture or envelope exceeds 8 MiB") + } + return total, ignored, nil +} + +func envelope(v any, key []byte, remaining uint64, visit func(record) error) (uint64, uint64, error) { + root, err := object(v) + if err != nil { + return 0, 0, err + } + if _, ok := root["resourceLogs"]; !ok { + return 0, 0, errors.New("expected resourceLogs; use a collector JSON logs export") + } + resources, err := children(root, "resourceLogs", 1024) + if err != nil { + return 0, 0, err + } + var total, ignored uint64 + for _, rv := range resources { + r, err := object(rv) + if err != nil { + return total, ignored, err + } + rm := map[string]any{} + if x, ok := r["resource"]; ok { + rm, err = object(x) + if err != nil { + return total, ignored, err + } + } + ra, err := attributes(rm) + if err != nil { + return total, ignored, err + } + scopes, err := children(r, "scopeLogs", 1024) + if err != nil { + return total, ignored, err + } + for _, sv := range scopes { + s, err := object(sv) + if err != nil { + return total, ignored, err + } + logs, err := children(s, "logRecords", 100000) + if err != nil { + return total, ignored, err + } + for _, lv := range logs { + if total >= remaining { + return total, ignored, errors.New("capture exceeds 1000000 log records") + } + total++ + l, err := object(lv) + if err != nil { + return total, ignored, err + } + body, _ := l["body"].(map[string]any) + event, _ := body["stringValue"].(string) + if len(body) != 1 || event != "claude_code.api_request" && event != "claude_code.api_error" && event != "claude_code.api_retries_exhausted" { + ignored++ + continue + } + a, err := attributes(l) + if err != nil { + return total, ignored, err + } + var stamp string + switch x := l["timeUnixNano"].(type) { + case string: + stamp = x + case json.Number: + stamp = x.String() + } + n, err := strconv.ParseInt(stamp, 10, 64) + if err != nil || n <= 0 || len(stamp) > 19 { + return total, ignored, errors.New("recognized log needs a positive source timeUnixNano within signed 64-bit range") + } + original, _ := json.Marshal([]any{rm, s["scope"], r["schemaUrl"], s["schemaUrl"], l}) + row := record{event: event, time: time.Unix(0, n).UTC(), attrs: a, resource: ra, fingerprint: attribution.Digest(key, "claude-records/exact-copy/v1", original)} + if err := visit(row); err != nil { + return total, ignored, err + } + } + } + } + return total, ignored, nil +} + +// resolve compares locality before choosing; a conflicting identity is unknown. +func (r record) resolve(name string) (value, bool) { + a, ap := r.attrs[name] + b, bp := r.resource[name] + if ap && bp && a != b { + return value{invalid: true}, true + } + if ap { + return a, true + } + return b, bp +} + +func (r record) text(name string) (string, bool) { + v, ok := r.resolve(name) + if !ok || v.invalid || v.kind != "stringValue" || strings.TrimSpace(v.text) == "" { + return "", false + } + return v.text, true +} diff --git a/internal/clauderecords/report.go b/internal/clauderecords/report.go new file mode 100644 index 0000000..f9c3cde --- /dev/null +++ b/internal/clauderecords/report.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package clauderecords + +import ( + "math/big" + + "github.com/llm-measurement/fleetdiff/internal/accounting" + "github.com/llm-measurement/fleetdiff/internal/compare" + "github.com/llm-measurement/fleetdiff/internal/spend" + "github.com/llm-measurement/llm-sketchkit/go/sketchkit/frequentitems" + sketchhash "github.com/llm-measurement/llm-sketchkit/go/sketchkit/hash" +) + +type Measured struct { + Value *uint64 `json:"tokens"` + Records uint64 `json:"covered_records"` +} +type Cache struct { + Input Measured `json:"input"` + Read Measured `json:"read"` + Write Measured `json:"write"` + Uncached Measured `json:"uncached_input"` + ReadShare *float64 `json:"read_share"` + WriteShare *float64 `json:"write_share"` +} +type AttributionCoverage struct { + Records uint64 `json:"records"` + TokenRecords uint64 `json:"token_covered_records"` + CostRecords uint64 `json:"cost_covered_records"` +} +type Window struct { + Period spend.Period `json:"period"` + Records uint64 `json:"observed_api_request_records"` + Input Measured `json:"inclusive_input"` + Output Measured `json:"output"` + Total Measured `json:"complete_total"` + Cache Cache `json:"cache"` + Cost *string `json:"recorded_cost_usd"` + CostRecords uint64 `json:"cost_covered_records"` + CostRepresentations map[string]uint64 `json:"cost_source_types"` + DecimalMinusMicros *string `json:"decimal_minus_micros_usd"` + PairedCosts uint64 `json:"paired_cost_records"` + Unattributed uint64 `json:"unattributed_records"` + IdentityKinds map[string]uint64 `json:"identity_kinds"` + IdentityCoverage map[string]AttributionCoverage `json:"identity_coverage"` + Quality map[string]uint64 `json:"quality"` + ErrorEvents uint64 `json:"api_error_events"` + ErrorStatus map[string]uint64 `json:"api_error_status_categories"` + ExhaustedEvents uint64 `json:"api_retries_exhausted_events"` + DuplicateCopies uint64 `json:"exact_copies_after_first"` + PromptRecords uint64 `json:"prompt_attributed_records"` + Prompts uint64 `json:"observed_scoped_prompts"` + RecordsPerPrompt *float64 `json:"records_per_scoped_prompt"` + MultipleRecordPrompts uint64 `json:"prompts_with_multiple_records"` + Versions map[string]uint64 `json:"version_mix"` + Profiles map[string]uint64 `json:"capture_profile_mix"` +} +type Model struct { + Name string `json:"model"` + Before Window `json:"before"` + After Window `json:"after"` +} +type Decomposition struct { + RecordContribution float64 `json:"record_count_contribution"` + SizeContribution float64 `json:"tokens_per_record_contribution"` +} +type Report struct { + Schema string `json:"schema"` + Unit string `json:"unit"` + Finality string `json:"finality"` + AccountingID string `json:"accounting_contract"` + Hashing string `json:"hashing"` + GroupBy string `json:"group_by"` + Deduplication string `json:"deduplication"` + PeriodSelection string `json:"period_selection"` + InputFiles int `json:"input_files"` + ReadRecords uint64 `json:"read_log_records"` + IgnoredEvents uint64 `json:"ignored_events"` + OutsidePeriods uint64 `json:"outside_periods"` + Before Window `json:"before"` + After Window `json:"after"` + Models []Model `json:"models"` + Rankings compare.Concentration `json:"token_growth"` + UncachedRankings compare.Concentration `json:"uncached_input_growth"` + ComparisonChecks []string `json:"comparison_checks"` + Decomposition *Decomposition `json:"record_decomposition"` +} + +type aggregate struct { + w Window + cost big.Rat + costDelta big.Rat + top, cacheTop *frequentitems.Sketch +} + +func newAggregate(p spend.Period, rank bool) *aggregate { + a := &aggregate{w: Window{Period: p, Quality: map[string]uint64{}, IdentityKinds: map[string]uint64{}, IdentityCoverage: map[string]AttributionCoverage{}, Versions: map[string]uint64{}, Profiles: map[string]uint64{}, CostRepresentations: map[string]uint64{}, ErrorStatus: map[string]uint64{}}} + if rank { + a.top, _ = frequentitems.New(frequentitems.ProfileSmall, sketchhash.UserV1, sketchhash.HMACSHA25664) + a.cacheTop, _ = frequentitems.New(frequentitems.ProfileSmall, sketchhash.UserV1, sketchhash.HMACSHA25664) + } + return a +} +func addMeasured(m *Measured, n *uint64) error { + if n == nil { + return nil + } + var current uint64 + if m.Value != nil { + current = *m.Value + } + sum, err := accounting.AddBounded(current, *n) + if err != nil { + return err + } + m.Value = &sum + m.Records++ + return nil +} +func (a *aggregate) add(o observation) error { + a.w.Records++ + for _, m := range []struct { + dst *Measured + n *uint64 + }{{&a.w.Input, o.input}, {&a.w.Output, o.output}, {&a.w.Total, o.total}} { + if err := addMeasured(m.dst, m.n); err != nil { + return err + } + } + // Cache shares use one common covered population, never missing-as-zero. + if o.input != nil { + uncached := *o.input - *o.read + for _, m := range []struct { + dst *Measured + n *uint64 + }{{&a.w.Cache.Input, o.input}, {&a.w.Cache.Read, o.read}, {&a.w.Cache.Write, o.write}, {&a.w.Cache.Uncached, &uncached}} { + if err := addMeasured(m.dst, m.n); err != nil { + return err + } + } + } + if o.cost != nil { + a.cost.Add(&a.cost, o.cost) + a.w.CostRecords++ + a.w.CostRepresentations[o.costKind]++ + } + if o.costDelta != nil { + a.costDelta.Add(&a.costDelta, o.costDelta) + a.w.PairedCosts++ + } + return accounting.MergeCounts(a.w.Quality, o.quality) +} +func (a *aggregate) result() Window { + w := a.w + if w.CostRecords > 0 { + s := accounting.CostText(&a.cost) + w.Cost = &s + } + if w.PairedCosts > 0 { + s := accounting.CostText(&a.costDelta) + w.DecimalMinusMicros = &s + } + if w.Cache.Input.Value != nil && *w.Cache.Input.Value > 0 { + r := float64(*w.Cache.Read.Value) / float64(*w.Cache.Input.Value) + w.Cache.ReadShare = &r + v := float64(*w.Cache.Write.Value) / float64(*w.Cache.Input.Value) + w.Cache.WriteShare = &v + } + if w.Prompts > 0 { + n := float64(w.PromptRecords) / float64(w.Prompts) + w.RecordsPerPrompt = &n + } + return w +} diff --git a/internal/cli/claude.go b/internal/cli/claude.go new file mode 100644 index 0000000..01c6b1a --- /dev/null +++ b/internal/cli/claude.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/llm-measurement/fleetdiff/internal/clauderecords" + "github.com/llm-measurement/fleetdiff/internal/compare" +) + +func runClaudeRecords(paths []string, before, after, group, secret, scope, format string, top int, hashes, dedup bool, out, errout io.Writer) int { + if format != "text" && format != "json" || top < 1 || top > 100 || group != "user" && group != "session" { + fmt.Fprintln(errout, "Claude logs need text/json, group-by user/session and top 1-100") + return 2 + } + r, err := clauderecords.Investigate(paths, clauderecords.Options{BeforePeriod: before, AfterPeriod: after, GroupBy: group, SecretEnv: secret, SourceScope: scope, Top: top, ShowHashes: hashes, DeduplicateExact: dedup}) + if err != nil { + fmt.Fprintln(errout, err) + return 1 + } + var buf bytes.Buffer + if format == "json" { + e := json.NewEncoder(&buf) + e.SetIndent("", " ") + if err := e.Encode(r); err != nil { + fmt.Fprintln(errout, "cannot encode Claude log report") + return 1 + } + } else { + renderClaudeRecords(&buf, r) + } + if _, err := io.Copy(out, &buf); err != nil { + fmt.Fprintln(errout, "cannot write Claude log report") + return 1 + } + return 0 +} + +func measuredText(m clauderecords.Measured) string { + if m.Value == nil { + return "unknown" + } + return spendNumber(*m.Value) +} +func renderClaudeRecords(out io.Writer, r clauderecords.Report) { + a, b := r.Before, r.After + if len(r.ComparisonChecks) > 0 { + fmt.Fprintf(out, "Record comparability needs review: %s -> %s observed api_request records; covered token totals %s -> %s.\n", spendNumber(a.Records), spendNumber(b.Records), measuredText(a.Total), measuredText(b.Total)) + } else { + fmt.Fprintf(out, "Observed api_request token totals: %s -> %s, across %s -> %s records.\n", measuredText(a.Total), measuredText(b.Total), spendNumber(a.Records), spendNumber(b.Records)) + } + fmt.Fprintln(out, "Unit: observed api_request records; finality unknown. Cancelled work may leave no event; retries can produce several records.") + switch r.PeriodSelection { + case "complete_utc_weeks": + fmt.Fprintln(out, "Periods: the two most recent complete seven-day UTC periods within the observed timestamps; export coverage is unverified.") + case "observed_span_halves": + fmt.Fprintln(out, "Periods: two equal halves of the file set's observed span (fewer than 14 complete UTC days); export coverage is unverified.") + } + for _, p := range []struct { + name string + w clauderecords.Window + }{{"Before", a}, {"After", b}} { + fmt.Fprintf(out, "%s: %s to %s (end exclusive).\n", p.name, p.w.Period.Start.Format("Mon 2006-01-02 15:04:05.999999999 MST"), p.w.Period.End.Format("Mon 2006-01-02 15:04:05.999999999 MST")) + } + fmt.Fprintln(out, "\nWhat changed?") + for _, m := range r.Models { + fmt.Fprintf(out, " %s: %s -> %s covered tokens, %s -> %s records.\n", m.Name, measuredText(m.Before.Total), measuredText(m.After.Total), spendNumber(m.Before.Records), spendNumber(m.After.Records)) + } + if d := r.Decomposition; d != nil { + fmt.Fprintf(out, " Record-count contribution: %+.0f tokens; tokens-per-record contribution: %+.0f (arithmetic split).\n", d.RecordContribution, d.SizeContribution) + } + fmt.Fprintf(out, " Token coverage: %s of %s -> %s of %s records have all four components.\n", spendNumber(a.Total.Records), spendNumber(a.Records), spendNumber(b.Total.Records), spendNumber(b.Records)) + if a.Cost != nil || b.Cost != nil { + cost := func(s *string) string { + if s == nil { + return "unknown" + } + return "USD " + *s + } + fmt.Fprintf(out, " Recorded cost: %s -> %s (%s -> %s covered records; same unknown finality).\n", cost(a.Cost), cost(b.Cost), spendNumber(a.CostRecords), spendNumber(b.CostRecords)) + } + if a.Cache.Input.Value != nil || b.Cache.Input.Value != nil { + fmt.Fprintf(out, " Cache reads: %s of %s input tokens -> %s of %s (%s -> %s covered records).\n", measuredText(a.Cache.Read), measuredText(a.Cache.Input), measuredText(b.Cache.Read), measuredText(b.Cache.Input), spendNumber(a.Cache.Input.Records), spendNumber(b.Cache.Input.Records)) + fmt.Fprintf(out, " Cache writes: %s -> %s; uncached input: %s -> %s.\n", measuredText(a.Cache.Write), measuredText(b.Cache.Write), measuredText(a.Cache.Uncached), measuredText(b.Cache.Uncached)) + } + claudeRanking(out, "Who drove the observed token change?", r.Rankings, r.GroupBy) + claudeRanking(out, "Who drove the uncached input change?", r.UncachedRankings, r.GroupBy) + fmt.Fprintf(out, " Unattributed records: %s -> %s. Installation aliases identify installations, not people.\n", spendNumber(a.Unattributed), spendNumber(b.Unattributed)) + fmt.Fprintln(out, "\nComparison checks") + for _, c := range r.ComparisonChecks { + fmt.Fprintf(out, " %s\n", claudeCheckText(c)) + } + if len(r.ComparisonChecks) == 0 { + fmt.Fprintln(out, " Observed versions, capture profiles, attribution and record mix are consistent; this does not establish final usage or capture completeness.") + } + fmt.Fprintf(out, " Prompt-attributed records / scoped prompts: %s / %s -> %s / %s.\n", spendNumber(a.PromptRecords), spendNumber(a.Prompts), spendNumber(b.PromptRecords), spendNumber(b.Prompts)) + fmt.Fprintf(out, " api_error events: %s -> %s; retries_exhausted companions: %s -> %s (separate events, not additional requests).\n", spendNumber(a.ErrorEvents), spendNumber(b.ErrorEvents), spendNumber(a.ExhaustedEvents), spendNumber(b.ExhaustedEvents)) + keys := map[string]bool{} + for k := range a.Quality { + keys[k] = true + } + for k := range b.Quality { + keys[k] = true + } + var names []string + for k := range keys { + names = append(names, k) + } + sort.Strings(names) + for _, k := range names { + if k == "cost_decimal" { + continue + } + if a.Quality[k]+b.Quality[k] > 0 { + fmt.Fprintf(out, " %s: %s -> %s records.\n", strings.ReplaceAll(k, "_", " "), spendNumber(a.Quality[k]), spendNumber(b.Quality[k])) + } + } + if a.DuplicateCopies+b.DuplicateCopies > 0 { + fmt.Fprintf(out, " Exact exported copies after the first: %s -> %s; %s.\n", spendNumber(a.DuplicateCopies), spendNumber(b.DuplicateCopies), r.Deduplication) + } + fmt.Fprintln(out, "\nNext: check record mix and capture coverage before interpreting this as a workload change. Use native receipts for final usage.") +} +func claudeRanking(out io.Writer, title string, c compare.Concentration, group string) { + fmt.Fprintln(out, "\n"+title) + if len(c.Movers) == 0 { + fmt.Fprintln(out, " No attributed token weights available.") + return + } + for _, m := range c.Movers { + fmt.Fprintf(out, " %s: %s -> %s tokens (%s).", m.Item, spendInterval(m.Before), spendInterval(m.After), signedInterval(m.Delta)) + if m.Hash != "" { + fmt.Fprintf(out, " hash=%s", m.Hash) + } + fmt.Fprintln(out) + } + fmt.Fprintf(out, " Grouping: %s; sessions are scoped to the available account or installation identity.\n", group) +} +func claudeCheckText(code string) string { + return map[string]string{ + "error_status_mix_changed": "The mix of rate-limit, server and other API errors changed.", + "period_without_api_request_records": "One period has no api_request records: missing observations are not zero usage.", + "incomplete_token_components": "Token components are incomplete or invalid; totals cover only complete records.", + "unverified_source_profile": "Source version or capture profile differs from the tested 2.1.296 recipe, or is missing.", + "source_version_mix_changed": "Source-version mix changed.", "capture_profile_mix_changed": "Capture-profile mix changed.", + "identity_source_mix_changed": "Account/email/installation attribution mix changed.", "incomplete_prompt_identity": "Some records lack an owner, session or prompt identity for the per-prompt check.", + "records_per_prompt_changed": "Records per scoped prompt changed; retry or request mix may differ.", "error_or_exhaustion_mix_changed": "Error or retry-exhaustion event counts changed relative to observed records.", + "attribution_coverage_changed": "Attribution coverage changed.", "exact_export_copies_retained": "Exact exported copies were retained; use --deduplicate-exact only for overlapping exports.", + }[code] +} diff --git a/internal/cli/claude_test.go b/internal/cli/claude_test.go new file mode 100644 index 0000000..4658b7b --- /dev/null +++ b/internal/cli/claude_test.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +func claudeCapture(t *testing.T) string { + t.Helper() + attr := func(key string, v any) any { + return map[string]any{"key": key, "value": map[string]any{"stringValue": v}} + } + var logs []any + for _, day := range []int{11, 12} { + logs = append(logs, map[string]any{"timeUnixNano": strconv.FormatInt(time.Date(2026, 10, day, 0, 0, 0, 0, time.UTC).UnixNano(), 10), "body": map[string]any{"stringValue": "claude_code.api_request"}, "attributes": []any{ + attr("input_tokens", "7"), attr("cache_read_tokens", "11"), attr("cache_creation_tokens", "5"), attr("output_tokens", "4"), attr("cost_usd", "0.00003435"), attr("cost_usd_micros", "34"), attr("user.id", "PRIVATE_INSTALL_ID"), attr("session.id", "PRIVATE_SESSION_ID"), attr("prompt.id", "PRIVATE_PROMPT_ID"), attr("model", "PRIVATE_MODEL"), attr("error", "SECRET_ERROR /Users/private/code.py def private_function():"), attr("unknown", "SECRET_UNKNOWN"), + }}) + } + b, e := json.Marshal(map[string]any{"resourceLogs": []any{map[string]any{"resource": map[string]any{"attributes": []any{attr("service.version", "2.1.296"), attr("fleetdiff.capture.profile", "claude-code-logs/v1"), attr("process.command_line", "SECRET_COMMAND")}}, "scopeLogs": []any{map[string]any{"scope": map[string]any{"name": "SECRET_SCOPE", "attributes": []any{attr("code", "SECRET_CODE")}}, "logRecords": logs}}}}}) + if e != nil { + t.Fatal(e) + } + p := filepath.Join(t.TempDir(), "SECRET_FILENAME.jsonl") + if e = os.WriteFile(p, b, 0600); e != nil { + t.Fatal(e) + } + return p +} +func TestClaudeCLIPrivacyAndObservationUnit(t *testing.T) { + p := claudeCapture(t) + for _, format := range []string{"text", "json"} { + var out, err bytes.Buffer + code := Run([]string{"investigate", "--claude-code-logs", p, "--before-period", "2026-10-11", "--after-period", "2026-10-12", "--format", format}, &out, &err) + if code != 0 { + t.Fatalf("%d: %s", code, &err) + } + for _, s := range []string{"PRIVATE_", "SECRET_", "/Users/private", "private_function", "logical API", "success", "\"requests\""} { + if strings.Contains(out.String()+err.String(), s) { + t.Fatalf("unsafe label or leaked input %q: %s", s, &out) + } + } + if !strings.Contains(out.String(), "observed api_request records") || !strings.Contains(out.String(), "unknown") { + t.Fatal(out.String()) + } + if format == "text" && !strings.HasPrefix(out.String(), "Observed api_request token totals: 27 -> 27") { + t.Fatal(out.String()) + } + } +} +func TestClaudeCLIRejectsMixedModesAtomically(t *testing.T) { + p := claudeCapture(t) + for _, extra := range [][]string{{"--litellm-spend", "SECRET_FILE"}, {"--expected", "SECRET_PRODUCER"}, {"--before", "SECRET_PATH"}, {"--period", "2026-10-11"}, {"--session-budget-usd", "1"}, {"--group-by", "key"}, {"--format", "SECRET_FORMAT"}, {"--hash-secret-env", "SECRET_ENV"}} { + args := []string{"investigate", "--claude-code-logs", p, "--before-period", "2026-10-11", "--after-period", "2026-10-12"} + args = append(args, extra...) + var out, err bytes.Buffer + if Run(args, &out, &err) == 0 || out.Len() != 0 || strings.Contains(err.String(), "SECRET_") { + t.Fatalf("bad failure: %s / %s", &out, &err) + } + } + for _, b := range []string{`{"resourceLogs":["SECRET_ERROR"]}`, `{"resourceLogs":[],"error":"SECRET_ERROR"}`} { + if e := os.WriteFile(p, []byte(b), 0600); e != nil { + t.Fatal(e) + } + var out, err bytes.Buffer + if Run([]string{"investigate", "--claude-code-logs", p, "--before-period", "2026-10-11", "--after-period", "2026-10-12"}, &out, &err) == 0 || out.Len() != 0 || strings.Contains(err.String(), "SECRET_") { + t.Fatalf("bad failure: %s / %s", &out, &err) + } + } +} + +func TestClaudeRotatedCLIOptionalPeriodsAndSafeErrors(t *testing.T) { + p := claudeCapture(t) + for _, format := range []string{"text", "json"} { + args := []string{"investigate", "--claude-code-logs", p, "--claude-code-logs", p, "--deduplicate-exact", "--format", format} + var out, errout bytes.Buffer + if code := Run(args, &out, &errout); code != 0 { + t.Fatal(code, errout.String()) + } + if strings.Contains(out.String()+errout.String(), "SECRET_") || strings.Contains(out.String(), "PRIVATE_") { + t.Fatal("input leaked") + } + if format == "json" { + var r struct { + Files int `json:"input_files"` + Selection string `json:"period_selection"` + Before, After struct { + Count int `json:"observed_api_request_records"` + } + } + if err := json.Unmarshal(out.Bytes(), &r); err != nil { + t.Fatal(err) + } + if r.Files != 2 || r.Selection != "observed_span_halves" || r.Before.Count != 1 || r.After.Count != 1 { + t.Fatal(out.String()) + } + } else { + for _, s := range []string{"file set's observed span", "Sun 2026-10-11", "Mon 2026-10-12 00:00:00.000000002 UTC"} { + if !strings.Contains(out.String(), s) { + t.Fatal(s, out.String()) + } + } + } + } + for _, extra := range [][]string{{"--claude-code-logs", "SECRET_MISSING"}, {"--claude-code-logs", ""}, {"--before-period", "2026-10-11"}} { + var out, errout bytes.Buffer + if code := Run(append([]string{"investigate", "--claude-code-logs", p}, extra...), &out, &errout); code == 0 || out.Len() != 0 || strings.Contains(errout.String(), "SECRET_") { + t.Fatal(code, out.String(), errout.String()) + } + } +} diff --git a/internal/cli/run.go b/internal/cli/run.go index e913da6..41dfc3b 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -43,6 +43,7 @@ Usage: fleetdiff investigate --before PATH --after PATH --expected PRODUCER[,PRODUCER] [options] fleetdiff investigate --litellm-spend FILE [--before-period DATE --after-period DATE] [options] fleetdiff investigate --litellm-spend FILE --session-budget-usd USD [--period DATE] + fleetdiff investigate --claude-code-logs FILE [--claude-code-logs FILE ...] [options] fleetdiff compare --before PATH --after PATH --expected PRODUCER,PRODUCER [options] PATH is one canonical summary JSON file or a directory of summary JSON files. @@ -59,13 +60,16 @@ Options: --allow-partial Permit missing producers or incomplete observation intervals --show-hashes Include pseudonymous, linkable hashes in output --litellm-spend FILE Read a request-level SQL export: CSV, JSON array or JSONL + --claude-code-logs FILE Read collector OTLP JSONL; repeat for rotated files + --source-scope NAME One application namespace (required with a persistent Claude hashing key) + --deduplicate-exact Remove exact exported copies only (Claude logs; default retains copies) --before-period PERIOD UTC date or RFC3339 start/end (requires after-period) --after-period PERIOD UTC date or RFC3339 start/end (equal duration) --group-by KIND key (default), team, user, end-user or session --period PERIOD Single-period session review: UTC date or RFC3339 start/end --session-budget-usd USD Review observed session totals against a supplied cap --session-counter-window DURATION Compare positive-spend spans with this duration (default 1h) - --hash-secret-env NAME Use an existing secret for repeat spend comparisons + --hash-secret-env NAME Use an existing secret for repeat file comparisons --help Show this help --version Print version, revision, Go toolchain, and platform @@ -75,6 +79,9 @@ Runs offline using local files. Default output uses aliases for identities. Spend periods default to the latest two complete 7-day UTC intervals in the file. With fewer than 14 complete days, supply both period flags. Daily UI aggregates need a request-level SQL export; see examples/litellm-spend/README.md. +Claude logs default to complete UTC weeks, or two labeled observed-span halves +for shorter histories. Period flags override this; grouping defaults to user (or session). +They count observed api_request records, with finality unknown; see examples/claude-code-logs/README.md. Exit status: 0 report/help, 1 input/comparison/output error, 2 invalid command/options. diagnose and scan additionally use 3 for findings and 4 for unavailable evidence. ` @@ -124,6 +131,16 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { partial := flags.Bool("allow-partial", false, "") hashes := flags.Bool("show-hashes", false, "") spendPath := flags.String("litellm-spend", "", "") + var claudePaths []string + flags.Func("claude-code-logs", "", func(path string) error { + if path == "" || len(claudePaths) >= 256 { + return errors.New("select 1-256 collector JSONL files") + } + claudePaths = append(claudePaths, path) + return nil + }) + sourceScope := flags.String("source-scope", "", "") + deduplicate := flags.Bool("deduplicate-exact", false, "") beforePeriod := flags.String("before-period", "", "") afterPeriod := flags.String("after-period", "", "") group := flags.String("group-by", "key", "") @@ -145,6 +162,27 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { } return fail(2, "invalid compare options; run fleetdiff compare --help") } + claudeMode, claudeIncompatible, claudeGroup := false, false, false + flags.Visit(func(f *flag.Flag) { + switch f.Name { + case "claude-code-logs", "source-scope", "deduplicate-exact": + claudeMode = true + case "group-by": + claudeGroup = true + case "before-period", "after-period", "hash-secret-env", "format", "top", "show-hashes": + default: + claudeIncompatible = true + } + }) + if claudeMode { + if args[0] != "investigate" || len(claudePaths) == 0 || claudeIncompatible || flags.NArg() != 0 { + return fail(2, "use investigate --claude-code-logs FILE; do not mix with spend, session-budget or summary options") + } + if !claudeGroup { + *group = "user" + } + return runClaudeRecords(claudePaths, *beforePeriod, *afterPeriod, *group, *secret, *sourceScope, *format, *top, *hashes, *deduplicate, out, errout) + } spendMode, incompatible := false, false sessionMode, windowSet, emptySessionOption, twoPeriods := false, false, false, false groupExplicit := false diff --git a/scripts/restricted-smoke.sh b/scripts/restricted-smoke.sh index 5f3df57..0aa2f8c 100644 --- a/scripts/restricted-smoke.sh +++ b/scripts/restricted-smoke.sh @@ -20,9 +20,11 @@ tar -xzf "$archive" -C "$work" fleetdiff chmod 755 "$work/fleetdiff" CGO_ENABLED=0 GOOS=linux GOARCH=${platform#linux/} go build -o "$work/network-probe" ./scripts/testdata/network-probe go run ./examples/scan --out "$work/windows" +python3 examples/claude-code-logs/fixtures.py > "$work/claude-records.jsonl" # Only synthetic fixtures and a test-only executable are exposed to the container. chmod 755 "$work/network-probe" "$work/windows" chmod 644 "$work/windows/"*.json +chmod 644 "$work/claude-records.jsonl" cp scripts/restricted.Dockerfile "$work/Dockerfile" # FROM scratch has no external base to download or mutable tag to resolve. # Pin both runtime modes to exactly the same locally built image ID. @@ -40,6 +42,7 @@ run_container() { --mount "type=bind,src=$(pwd)/examples,dst=/examples,readonly" \ --mount "type=bind,src=$(pwd)/testdata,dst=/testdata,readonly" \ --mount "type=bind,src=$work/windows,dst=/windows,readonly" \ + --mount "type=bind,src=$work/claude-records.jsonl,dst=/claude-records.jsonl,readonly" \ --mount "type=bind,src=$work/network-probe,dst=/network-probe,readonly" \ --entrypoint "$entrypoint" "$image" "$@" } @@ -95,6 +98,20 @@ check_report investigate-spend '.before.recorded_tokens == 8000 and .after.recor .concentration_of_increase.tracked_contributors == 2 and .concentration_of_increase.share_of_net_recorded_increase.lower == 1' +check_case investigate-claude 0 investigate --claude-code-logs /claude-records.jsonl \ + --before-period 2026-10-11 --after-period 2026-10-12 --source-scope offline-test \ + --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json +check_report investigate-claude '.unit == "observed api_request records" and .finality == "unknown" and + .before.complete_total.tokens == 27 and .after.complete_total.tokens == 27 and + .after.api_error_events == 1 and .after.api_retries_exhausted_events == 1' + +check_case investigate-claude-rotated 0 investigate --claude-code-logs /claude-records.jsonl \ + --claude-code-logs /claude-records.jsonl --deduplicate-exact --source-scope offline-test \ + --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json +check_report investigate-claude-rotated '.input_files == 2 and .period_selection == "observed_span_halves" and + .before.observed_api_request_records == 1 and .after.observed_api_request_records == 1 and + .before.complete_total.tokens == 27 and .after.complete_total.tokens == 27' + check_case session-budget 0 investigate --litellm-spend /examples/litellm-spend/session-budget.csv \ --period 2026-10-03 --group-by session --session-budget-usd 1.00 \ --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json @@ -111,4 +128,4 @@ check_report diagnose '.status == "supported_safe" and .configuration_parsed == check_case scan 3 scan /windows --expected app --recent 2 --as-of 2026-10-04T00:30:00Z --format json check_report scan '.schema == "fleetdiff-scan/v1" and .unusual_windows == 2' -printf 'PASS restricted offline binary smoke: 7 cases (%s)\n' "$platform" +printf 'PASS restricted offline binary smoke: 9 cases (%s)\n' "$platform" From 9a8aef459f4a1b3b5ae5961304245b5cbb66e9c8 Mon Sep 17 00:00:00 2001 From: Vijay Date: Tue, 13 Oct 2026 19:02:11 -0400 Subject: [PATCH 3/3] test: validate private rotation fixtures as their owning user Signed-off-by: Vijay --- examples/claude-code-logs/test_rotation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/claude-code-logs/test_rotation.py b/examples/claude-code-logs/test_rotation.py index fb99ebd..84c78f5 100644 --- a/examples/claude-code-logs/test_rotation.py +++ b/examples/claude-code-logs/test_rotation.py @@ -83,6 +83,7 @@ class RotationTests(unittest.TestCase): def validate(self, overlay): return subprocess.run([ "docker", "run", "--rm", "--pull", "never", "--network", "none", + "--user", f"{os.getuid()}:{os.getgid()}", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--env", "CLAUDE_CAPTURE_DIR=/capture", "--mount", f"type=bind,src={ROOT / 'collector.yaml'},dst=/etc/base.yaml,readonly",